Added some DB-access routines to help rsync keep extra filesystem info
about the files it is dealing with. This adds both the --db=CONFIG_FILE
option and the "db config" daemon parameter.
Future improvements may include:
- Updating of MD4 checksums when transferring any file, even w/o -c.
To make that work we'd need to make the sender force checksum_seed to
0 when using a DB and having the receiving side check to see if it
got a 0 checksum_seed.
- Caching of path info that allows for the finding of files to use for
moving/linking/copying/alternate-basis-use.
- Extend DB support beyond MySQL and SQLite (PostgreSQL?).
To use this patch, run these commands for a successful build:
patch -p1 <patches/db.diff
./prepare-source
./configure
make
based-on: e94bad1c156fc3910f24e2b3b71a81b0b0bdeb70
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@ aclocal.m4
/getgroups
/gmon.out
/rsync
+/rsyncdb
/stunnel-rsyncd.conf
/shconfig
/git-version.h
diff --git a/Makefile.in b/Makefile.in
--- a/Makefile.in
+++ b/Makefile.in
@@ -4,6 +4,7 @@ prefix=@prefix@
datarootdir=@datarootdir@
exec_prefix=@exec_prefix@
bindir=@bindir@
+sbindir=@sbindir@
libdir=@libdir@/rsync
mandir=@mandir@
@@ -33,7 +34,7 @@ SIMD_x86_64=simd-checksum-x86_64.o
ASM_x86_64=lib/md5-asm-x86_64.o
GENFILES=configure.sh aclocal.m4 config.h.in rsync.1 rsync.1.html \
- rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html
+ rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html rsyncdb.1 rsyncdb.1.html
HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
lib/pool_alloc.h lib/mdigest.h lib/md-defines.h version.h
LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
@@ -43,7 +44,7 @@ zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
util.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
- usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
+ usage.o fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
OBJS3=progress.o pipe.o @ASM@
DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
@@ -75,10 +76,12 @@ install: all
-$(MKDIR_P) $(DESTDIR)$(bindir)
$(INSTALLCMD) $(INSTALL_STRIP) -m 755 rsync$(EXEEXT) $(DESTDIR)$(bindir)
$(INSTALLCMD) -m 755 $(srcdir)/rsync-ssl $(DESTDIR)$(bindir)
+ rsync -ilt rsyncdb$(EXEEXT) $(DESTDIR)$(bindir)/
-$(MKDIR_P) $(DESTDIR)$(mandir)/man1
-$(MKDIR_P) $(DESTDIR)$(mandir)/man5
if test -f rsync.1; then $(INSTALLMAN) -m 644 rsync.1 $(DESTDIR)$(mandir)/man1; fi
if test -f rsync-ssl.1; then $(INSTALLMAN) -m 644 rsync-ssl.1 $(DESTDIR)$(mandir)/man1; fi
+ if test -f rsyncdb.1; then $(INSTALLMAN) -m 644 rsyncdb.1 $(DESTDIR)$(mandir)/man1; fi
if test -f rsyncd.conf.5; then $(INSTALLMAN) -m 644 rsyncd.conf.5 $(DESTDIR)$(mandir)/man5; fi
install-ssl-daemon: stunnel-rsyncd.conf
@@ -96,10 +99,13 @@ install-strip:
rsync$(EXEEXT): $(OBJS)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
+rsyncdb$(EXEEXT): rsync$(EXEEXT)
+ ln -s rsync$(EXEEXT) rsyncdb$(EXEEXT)
+
$(OBJS): $(HEADERS)
$(CHECK_OBJS): $(HEADERS)
tls.o xattrs.o: lib/sysxattrs.h
-usage.o: latest-year.h help-rsync.h help-rsyncd.h git-version.h default-cvsignore.h
+usage.o: latest-year.h help-rsync.h help-rsyncd.h help-rsyncdb.h git-version.h default-cvsignore.h
loadparm.o: default-dont-compress.h daemon-parm.h
flist.o: rounding.h
@@ -110,6 +116,9 @@ default-cvsignore.h default-dont-compress.h: rsync.1.md define-from-md.awk
help-rsync.h help-rsyncd.h: rsync.1.md help-from-md.awk
$(AWK) -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md
+help-rsyncdb.h: rsyncdb.1.md help-from-md.awk
+ awk -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsyncdb.1.md
+
daemon-parm.h: daemon-parm.txt daemon-parm.awk
$(AWK) -f $(srcdir)/daemon-parm.awk $(srcdir)/daemon-parm.txt
@@ -240,7 +249,7 @@ proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h
$(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h
.PHONY: man
-man: rsync.1 rsync-ssl.1 rsyncd.conf.5
+man: rsync.1 rsync-ssl.1 rsyncd.conf.5 rsyncdb.1
rsync.1: rsync.1.md md2man version.h Makefile
@$(srcdir)/maybe-make-man $(srcdir) rsync.1.md
@@ -251,9 +260,12 @@ rsync-ssl.1: rsync-ssl.1.md md2man version.h Makefile
rsyncd.conf.5: rsyncd.conf.5.md md2man version.h Makefile
@$(srcdir)/maybe-make-man $(srcdir) rsyncd.conf.5.md
+rsyncdb.1: rsyncdb.1.md md2man NEWS.md Makefile
+ @$(srcdir)/maybe-make-man $(srcdir) rsyncdb.1.md
+
.PHONY: clean
clean: cleantests
- rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \
+ rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) rsyncdb$(EXEEXT) \
rounding rounding.h *.old rsync*.1 rsync*.5 rsync*.html \
daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp
diff --git a/checksum.c b/checksum.c
--- a/checksum.c
+++ b/checksum.c
@@ -40,6 +40,7 @@ extern int whole_file;
extern int checksum_seed;
extern int protocol_version;
extern int proper_seed_order;
+extern int use_db;
extern const char *checksum_choice;
struct name_num_obj valid_checksums = {
@@ -386,6 +387,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
MD5_Update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder);
MD5_Final((uchar *)sum, &m5);
+ if (use_db)
+ db_set_checksum(5, st_p, sum);
break;
}
case CSUM_MD4:
@@ -425,6 +428,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
mdfour_result(&m, (uchar *)sum);
+ if (use_db)
+ db_set_checksum(4, st_p, sum);
break;
}
default:
diff --git a/cleanup.c b/cleanup.c
--- a/cleanup.c
+++ b/cleanup.c
@@ -28,6 +28,7 @@ extern int am_daemon;
extern int am_receiver;
extern int am_sender;
extern int io_error;
+extern int use_db;
extern int keep_partial;
extern int got_xfer_error;
extern int protocol_version;
@@ -143,6 +144,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
#include "case_N.h"
switch_step++;
+ if (use_db)
+ db_disconnect(False);
+
+ /* FALLTHROUGH */
+#include "case_N.h"
+
if (cleanup_child_pid != -1) {
int status;
int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
diff --git a/clientserver.c b/clientserver.c
--- a/clientserver.c
+++ b/clientserver.c
@@ -44,12 +44,15 @@ extern int numeric_ids;
extern int filesfrom_fd;
extern int remote_protocol;
extern int protocol_version;
+extern int always_checksum;
+extern int db_lax;
extern int io_timeout;
extern int no_detach;
extern int write_batch;
extern int default_af_hint;
extern int logfile_format_has_i;
extern int logfile_format_has_o_or_i;
+extern char *db_config;
extern char *bind_address;
extern char *config_file;
extern char *logfile_format;
@@ -809,6 +812,11 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
log_init(1);
+ if (*lp_db_config(i)) {
+ db_read_config(FLOG, lp_db_config(i));
+ db_lax = lp_db_lax(i);
+ }
+
#ifdef HAVE_PUTENV
if ((*lp_early_exec(module_id) || *lp_prexfer_exec(module_id)
|| *lp_postxfer_exec(module_id) || *lp_name_converter(module_id))
@@ -1021,6 +1029,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
am_server = 1; /* Don't let someone try to be tricky. */
quiet = 0;
+ db_config = NULL;
+
if (lp_ignore_errors(module_id))
ignore_errors = 1;
if (write_batch < 0)
diff --git a/configure.ac b/configure.ac
--- a/configure.ac
+++ b/configure.ac
@@ -494,6 +494,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
unistd.h utime.h grp.h compat.h sys/param.h ctype.h sys/wait.h \
sys/ioctl.h sys/filio.h string.h stdlib.h sys/socket.h sys/mode.h \
sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
+ mysql/mysql.h sqlite3.h \
netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h dl.h \
popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netinet/ip.h \
@@ -1369,6 +1370,48 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
fi
fi
+AC_MSG_CHECKING([whether to include mysql DB support])
+AC_ARG_ENABLE(mysql,
+ AC_HELP_STRING([--disable-mysql],
+ [disable mysql DB support]))
+
+if test x"$enable_mysql" = x"no"; then
+ AC_MSG_RESULT(no)
+else
+ AC_MSG_RESULT([yes])
+ AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0)
+ if test x$MYSQL_CONFIG = x1; then
+ AC_MSG_CHECKING(for mysql version >= 4)
+ mysql_version=`mysql_config --version`
+ mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
+ if test $mysql_major_version -lt 4; then
+ AC_MSG_RESULT(no.. skipping MySQL)
+ else
+ AC_MSG_RESULT(yes)
+
+ MYSQL_CFLAGS=`mysql_config --cflags`
+ MYSQL_LIBS=`mysql_config --libs`
+
+ CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
+ LIBS="$MYSQL_LIBS $LIBS"
+
+ AC_CHECK_LIB(mysqlclient, mysql_init)
+ fi
+ fi
+fi
+
+AC_MSG_CHECKING([whether to include sqlite DB support])
+AC_ARG_ENABLE(sqlite,
+ AC_HELP_STRING([--disable-sqlite],
+ [disable sqlite DB support]))
+
+if test x"$enable_sqlite" = x"no"; then
+ AC_MSG_RESULT(no)
+else
+ AC_CHECK_LIB(sqlite3, sqlite3_open)
+ AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2)
+fi
+
case "$CC" in
' checker'*|checker*)
AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
diff --git a/daemon-parm.txt b/daemon-parm.txt
--- a/daemon-parm.txt
+++ b/daemon-parm.txt
@@ -18,6 +18,7 @@ Locals: =================================================================
STRING auth_users NULL
STRING charset NULL
STRING comment NULL
+STRING db_config NULL
STRING dont_compress DEFAULT_DONT_COMPRESS
STRING early_exec NULL
STRING exclude NULL
@@ -51,6 +52,7 @@ INTEGER timeout 0
ENUM syslog_facility LOG_DAEMON
+BOOL db_lax False
BOOL fake_super False
BOOL forward_lookup True
BOOL ignore_errors False
diff --git a/db.c b/db.c
new file mode 100644
--- /dev/null
+++ b/db.c
@@ -0,0 +1,1943 @@
+/*
+ * Routines to access extended file info via DB.
+ *
+ * Copyright (C) 2008-2013 Wayne Davison
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, visit the http://fsf.org website.
+ */
+
+#include "rsync.h"
+#include "ifuncs.h"
+#include "itypes.h"
+#include "inums.h"
+#ifdef USE_OPENSSL
+#include "openssl/md4.h"
+#include "openssl/md5.h"
+#endif
+
+extern int recurse;
+extern int same_db;
+extern int am_receiver;
+extern int am_generator;
+extern int checksum_type;
+extern int db_clean, db_check, db_do_md4, db_do_md5, db_update, db_lax, db_init, db_mounts;
+extern int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
+extern int saw_db_output_opt, saw_db_sum_opt;
+extern char *db_config;
+
+#define MOUNT_HELPER_SCRIPT "/usr/sbin/rsyncdb-mountinfo"
+
+#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
+#define USE_MYSQL
+#include <mysql/mysql.h>
+#include <mysql/errmsg.h>
+#endif
+
+#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
+#define USE_SQLITE
+#include <sqlite3.h>
+#ifndef HAVE_SQLITE3_OPEN_V2
+#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \
+ sqlite3_open(dbname, dbhptr)
+#endif
+#ifndef HAVE_SQLITE3_PREPARE_V2
+#define sqlite3_prepare_v2 sqlite3_prepare
+#endif
+#define MAX_LOCK_FAILURES 10
+#define LOCK_FAIL_MSLEEP 100
+#endif
+
+#ifndef USE_OPENSSL
+#define MD5_CTX md_context
+#define MD5_Init md5_begin
+#define MD5_Update md5_update
+#define MD5_Final(digest, cptr) md5_result(cptr, digest)
+#endif
+
+#define DB_TYPE_NONE 0
+#define DB_TYPE_MYSQL 1
+#define DB_TYPE_SQLITE 2
+
+int use_db = DB_TYPE_NONE;
+int select_many_sums = 0;
+
+#define PREP_NORM 0
+#define PREP_MOUNT 1
+
+static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
+static unsigned int dbport = 0;
+static int transaction_state = -1;
+
+static union {
+#ifdef USE_MYSQL
+ MYSQL *mysql;
+#endif
+#ifdef USE_SQLITE
+ sqlite3 *sqlite;
+#endif
+ void *all;
+} dbh;
+
+#define SEL_DEV 0
+#define SEL_SUM 1
+#define REP_SUM 2
+#define UPD_CTIME 3
+#define INS_MOUNT 4
+#define UPD_MOUNT 5 /* SQLite only */
+#define SEL_MOUNT 6
+#define UN_MOUNT 7
+#define DEL_SUMS 8
+#define INS_PRESENT 9
+#define MAX_PREP_CNT 10
+
+#define MAX_BIND_CNT 7
+#define MAX_RESULT_BINDS 32
+
+static union {
+#ifdef USE_MYSQL
+ MYSQL_STMT *mysql;
+#endif
+#ifdef USE_SQLITE
+ sqlite3_stmt *sqlite;
+#endif
+ void *all;
+} statements[MAX_PREP_CNT];
+
+static int md_num;
+static enum logcode log_code;
+
+#ifdef USE_MYSQL
+static unsigned int bind_disk_id, bind_mdnum;
+static int64 bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
+static char bind_sum[MAX_DIGEST_LEN];
+static unsigned long result_length[MAX_RESULT_BINDS];
+static bool result_is_null[MAX_RESULT_BINDS], result_error[MAX_RESULT_BINDS];
+#elif defined USE_SQLITE
+static int64 bind_mtime;
+#endif
+static char bind_thishost[128+1];
+static unsigned long bind_thishost_len;
+static char *mount_helper_script = NULL;
+
+static char *error_log;
+#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
+static char bind_mount_uniq[128+1];
+static unsigned long bind_mount_uniq_len;
+static FILE *error_log_fp;
+#endif
+
+#define PTR_SIZE (sizeof (struct file_struct *))
+
+#if defined USE_MYSQL || defined USE_SQLITE
+static void update_mounts(void);
+#endif
+
+struct name_list {
+ struct name_list *next;
+ char name[1];
+} *dirs_list;
+
+int db_read_config(enum logcode code, const char *config_file)
+{
+ char buf[2048], *cp;
+ FILE *fp;
+ int lineno = 0;
+
+ log_code = code;
+
+ bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
+
+ if (!(fp = fopen(config_file, "r"))) {
+ rsyserr(log_code, errno, "unable to open %s", config_file);
+ return 0;
+ }
+ if (DEBUG_GTE(DB, 1))
+ rprintf(FCLIENT, "[%s] Reading DB config from %s\n", who_am_i(), config_file);
+ while (fgets(buf, sizeof buf, fp)) {
+ lineno++;
+ if ((cp = strchr(buf, '#')) == NULL
+ && (cp = strchr(buf, '\r')) == NULL
+ && (cp = strchr(buf, '\n')) == NULL)
+ cp = buf + strlen(buf);
+ while (cp != buf && isSpace(cp-1)) cp--;
+ *cp = '\0';
+
+ if (!*buf)
+ continue;
+
+ if (!(cp = strchr(buf, ':')))
+ goto invalid_line;
+ *cp++ = '\0';
+
+ while (isSpace(cp)) cp++;
+ if (strcasecmp(buf, "dbhost") == 0)
+ dbhost = strdup(cp);
+ else if (strcasecmp(buf, "dbuser") == 0)
+ dbuser = strdup(cp);
+ else if (strcasecmp(buf, "dbpass") == 0)
+ dbpass = strdup(cp);
+ else if (strcasecmp(buf, "dbname") == 0)
+ dbname = strdup(cp);
+ else if (strcasecmp(buf, "dbport") == 0)
+ dbport = atoi(cp);
+ else if (strcasecmp(buf, "transaction") == 0)
+ transaction_state = atoi(cp) ? 0 : -1;
+ else if (strcasecmp(buf, "mountHelper") == 0)
+ mount_helper_script = strdup(cp);
+ else if (strcasecmp(buf, "errlog") == 0)
+ error_log = strdup(cp);
+ else if (strcasecmp(buf, "thishost") == 0)
+ bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
+ else if (strcasecmp(buf, "dbtype") == 0) {
+#ifdef USE_MYSQL
+ if (strcasecmp(cp, "mysql") == 0) {
+ use_db = DB_TYPE_MYSQL;
+ continue;
+ }
+#endif
+#ifdef USE_SQLITE
+ if (strcasecmp(cp, "sqlite") == 0) {
+ use_db = DB_TYPE_SQLITE;
+ continue;
+ }
+#endif
+ rprintf(log_code,
+ "Unsupported dbtype on line #%d in %s.\n",
+ lineno, config_file);
+ use_db = DB_TYPE_NONE;
+ return 0;
+ } else {
+ invalid_line:
+ rprintf(log_code, "Invalid line #%d in %s\n",
+ lineno, config_file);
+ use_db = DB_TYPE_NONE;
+ return 0;
+ }
+ }
+ fclose(fp);
+
+ if (bind_thishost_len >= (int)sizeof bind_thishost)
+ bind_thishost_len = sizeof bind_thishost - 1;
+
+ if (!use_db || !dbname) {
+ rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
+ use_db = DB_TYPE_NONE;
+ return 0;
+ }
+
+ md_num = checksum_type == 5 ? 5 : 4;
+
+ if (error_log) {
+ if (use_db != DB_TYPE_SQLITE)
+ rprintf(log_code, "Ignoring errlog setting for non-SQLite DB.\n");
+#ifndef SQLITE_CONFIG_LOG
+ else
+ rprintf(log_code, "Your sqlite doesn't support SQLITE_CONFIG_LOG.\n");
+#endif
+ }
+
+ if (!mount_helper_script)
+ mount_helper_script = MOUNT_HELPER_SCRIPT;
+
+ return 1;
+}
+
+#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
+static void errorLogCallback(UNUSED(void *pArg), int iErrCode, const char *zMsg)
+{
+ fprintf(error_log_fp, "[%d] %s (%d)\n", (int)getpid(), zMsg, iErrCode);
+}
+#endif
+
+static int run_sql(const char *fmt, ...)
+{
+ va_list ap;
+ char *query;
+ int ok = 0, qlen;
+
+ va_start(ap, fmt);
+ qlen = vasprintf(&query, fmt, ap);
+ va_end(ap);
+ if (qlen < 0)
+ out_of_memory("run_sql");
+ if (DEBUG_GTE(DB, 3))
+ rprintf(FCLIENT, "[%s] SQL being run: %s\n", who_am_i(), query);
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ if (mysql_query(dbh.mysql, query) < 0) {
+ rprintf(FERROR, "Failed to run sql: %s\n", mysql_error(dbh.mysql));
+ rprintf(FERROR, "%s\n", query);
+ } else
+ ok = 1;
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc, lock_failures = 0;
+ while (1) {
+ if ((rc = sqlite3_exec(dbh.sqlite, query, NULL, NULL, NULL)) == 0)
+ break;
+ if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
+ break;
+ if (++lock_failures > MAX_LOCK_FAILURES)
+ break;
+ msleep(LOCK_FAIL_MSLEEP);
+ }
+ if (rc) {
+ rprintf(FERROR, "[%s] Failed to run sql: %s\n", who_am_i(), sqlite3_errmsg(dbh.sqlite));
+ rprintf(FERROR, "%s\n", query);
+ } else
+ ok = 1;
+ break;
+ }
+#endif
+ }
+
+ free(query);
+
+ return ok;
+}
+
+#ifdef USE_MYSQL
+static int prepare_mysql(int ndx, MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
+{
+ va_list ap;
+ char *query;
+ int qlen, param_cnt;
+ MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
+
+ if (stmt == NULL)
+ out_of_memory("prepare_mysql");
+
+ va_start(ap, fmt);
+ qlen = vasprintf(&query, fmt, ap);
+ va_end(ap);
+ if (qlen < 0)
+ out_of_memory("prepare_mysql");
+ if (DEBUG_GTE(DB, 3))
+ rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
+
+ if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
+ rprintf(log_code, "[%s] Prepare failed: %s\n", who_am_i(), mysql_stmt_error(stmt));
+ rprintf(log_code, "%s\n", query);
+ free(query);
+ return 0;
+ }
+
+ if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
+ rprintf(log_code, "[%s] Parameters in statement = %d, bind vars = %d\n",
+ who_am_i(), param_cnt, bind_cnt);
+ rprintf(log_code, "%s\n", query);
+ free(query);
+ return 0;
+ }
+ if (bind_cnt)
+ mysql_stmt_bind_param(stmt, binds);
+
+ statements[ndx].mysql = stmt;
+ free(query);
+
+ return 1;
+}
+#endif
+
+#ifdef USE_MYSQL
+static int prepare_mysql_queries(int type)
+{
+ MYSQL_BIND binds[MAX_BIND_CNT];
+ char *sql;
+
+ switch (type) {
+ case PREP_NORM:
+ sql="SELECT disk_id"
+ " FROM disk"
+ " WHERE host = ? AND devno = ?";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_STRING;
+ binds[0].buffer = &bind_thishost;
+ binds[0].buffer_length = bind_thishost_len;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_devno;
+ if (!prepare_mysql(SEL_DEV, binds, 2, sql))
+ return 0;
+
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_LONG;
+ binds[0].buffer = &bind_disk_id;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_ino;
+ if (select_many_sums) {
+ sql="SELECT checksum, sum_type, size, mtime, ctime"
+ " FROM inode_map"
+ " WHERE disk_id = ? AND ino = ?";
+ if (!prepare_mysql(SEL_SUM, binds, 2, sql))
+ return 0;
+ } else {
+ sql="SELECT checksum"
+ " FROM inode_map"
+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
+ " AND size = ? AND mtime = ? %s"; /* optional: AND ctime = ? */
+ binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[2].buffer = &bind_size;
+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[3].buffer = &bind_mtime;
+ if (!db_lax) {
+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[4].buffer = &bind_ctime;
+ }
+ if (!prepare_mysql(SEL_SUM, binds, 4 + !db_lax, sql, md_num, db_lax ? "" : "AND ctime = ?"))
+ return 0;
+ }
+
+ sql="INSERT INTO inode_map"
+ " SET disk_id = ?, ino = ?, sum_type = ?,"
+ " size = ?, mtime = ?, ctime = ?, checksum = ?"
+ " ON DUPLICATE KEY"
+ " UPDATE size = VALUES(size), mtime = VALUES(mtime),"
+ " ctime = VALUES(ctime), checksum = VALUES(checksum)";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_LONG;
+ binds[0].buffer = &bind_disk_id;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_ino;
+ binds[2].buffer_type = MYSQL_TYPE_LONG;
+ binds[2].buffer = &bind_mdnum;
+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[3].buffer = &bind_size;
+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[4].buffer = &bind_mtime;
+ binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[5].buffer = &bind_ctime;
+ binds[6].buffer_type = MYSQL_TYPE_BLOB;
+ binds[6].buffer = &bind_sum;
+ binds[6].buffer_length = MD5_DIGEST_LEN; /* Same as MD4_DIGEST_LEN */
+ if (!prepare_mysql(REP_SUM, binds, 7, sql))
+ return 0;
+
+ sql="UPDATE inode_map"
+ " SET ctime = ?"
+ " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[0].buffer = &bind_ctime;
+ binds[1].buffer_type = MYSQL_TYPE_LONG;
+ binds[1].buffer = &bind_disk_id;
+ binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[2].buffer = &bind_ino;
+ binds[3].buffer_type = MYSQL_TYPE_LONG;
+ binds[3].buffer = &bind_mdnum;
+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[4].buffer = &bind_size;
+ binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[5].buffer = &bind_mtime;
+ if (!prepare_mysql(UPD_CTIME, binds, 6, sql))
+ return 0;
+ break;
+
+ case PREP_MOUNT:
+ sql="INSERT INTO disk"
+ " SET host = ?, last_seen = ?, mount_uniq = ?, devno = ?"
+ " ON DUPLICATE KEY"
+ " UPDATE last_seen = VALUES(last_seen), devno = VALUES(devno)";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_STRING;
+ binds[0].buffer = &bind_thishost;
+ binds[0].buffer_length = bind_thishost_len;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_mtime; /* we abuse mtime to hold the last_seen value */
+ binds[2].buffer_type = MYSQL_TYPE_STRING;
+ binds[2].buffer = &bind_mount_uniq;
+ binds[2].buffer_length = sizeof bind_mount_uniq;
+ binds[2].length = &bind_mount_uniq_len;
+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[3].buffer = &bind_devno;
+ if (!prepare_mysql(INS_MOUNT, binds, 4, sql))
+ return 0;
+
+ sql="SELECT mount_uniq"
+ " FROM disk"
+ " WHERE host = ? AND last_seen < ? AND devno != 0";
+ /* Reusing first 2 binds from INS_MOUNT */
+ if (!prepare_mysql(SEL_MOUNT, binds, 2, sql))
+ return 0;
+
+ sql="UPDATE disk"
+ " SET devno = 0"
+ " WHERE host = ? AND last_seen < ? AND devno != 0";
+ /* Reusing binds from SEL_MOUNT */
+ if (!prepare_mysql(UN_MOUNT, binds, 2, sql))
+ return 0;
+ break;
+ }
+
+ return 1;
+}
+#endif
+
+#ifdef USE_MYSQL
+static int db_connect_mysql(void)
+{
+ const char *open_dbname = db_init ? "mysql" : dbname;
+
+ if (!(dbh.mysql = mysql_init(NULL)))
+ out_of_memory("db_read_config");
+
+ if (DEBUG_GTE(DB, 1)) {
+ rprintf(FCLIENT, "[%s] connecting: host=%s user=%s db=%s port=%d\n",
+ who_am_i(), dbhost, dbuser, open_dbname, dbport);
+ }
+ if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, open_dbname, dbport, NULL, 0)) {
+ rprintf(log_code, "[%s] Unable to connect to DB: %s\n", who_am_i(), mysql_error(dbh.mysql));
+ return 0;
+ }
+
+ if (db_init) {
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Creating DB %s (if it does not exist)\n", dbname);
+ if (!run_sql("CREATE DATABASE IF NOT EXISTS `%s`", dbname)
+ || !run_sql("USE `%s`", dbname))
+ exit_cleanup(RERR_IPC);
+
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Dropping old tables (if they exist))\n");
+ if (!run_sql("DROP TABLE IF EXISTS disk")
+ || !run_sql("DROP TABLE IF EXISTS inode_map"))
+ exit_cleanup(RERR_IPC);
+
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Creating empty tables ...\n");
+ if (!run_sql(
+ "CREATE TABLE disk (\n"
+ " disk_id integer unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,\n"
+ " host varchar(128) NOT NULL default 'localhost',\n"
+ " mount_uniq varchar(128) default NULL,\n"
+ " devno bigint unsigned NOT NULL,\n" /* This is 0 when not mounted */
+ " last_seen bigint NOT NULL,\n"
+ " UNIQUE KEY mount_lookup (host, mount_uniq),\n"
+ " KEY dev_lookup (devno, host)\n"
+ ")"))
+ exit_cleanup(RERR_IPC);
+
+ if (!run_sql(
+ "CREATE TABLE inode_map (\n"
+ " disk_id integer unsigned NOT NULL,\n"
+ " ino bigint unsigned NOT NULL,\n"
+ " sum_type tinyint NOT NULL default '0',\n"
+ " size bigint unsigned NOT NULL,\n"
+ " mtime bigint NOT NULL,\n"
+ " ctime bigint NOT NULL,\n"
+ " checksum binary(16) NOT NULL,\n"
+ " PRIMARY KEY (disk_id,ino,sum_type)\n"
+ ")"))
+ exit_cleanup(RERR_IPC);
+
+ if (!db_mounts)
+ exit_cleanup(0);
+ }
+
+ if (db_mounts) {
+ if (!prepare_mysql_queries(PREP_MOUNT))
+ exit_cleanup(RERR_IPC);
+ update_mounts();
+ exit_cleanup(0);
+ }
+
+ if (!prepare_mysql_queries(PREP_NORM))
+ return 0;
+
+ return 1;
+}
+#endif
+
+#ifdef USE_SQLITE
+static int prepare_sqlite(int ndx, const char *fmt, ...)
+{
+ va_list ap;
+ char *query;
+ int rc, qlen, lock_failures = 0;
+
+ va_start(ap, fmt);
+ qlen = vasprintf(&query, fmt, ap);
+ va_end(ap);
+ if (qlen < 0)
+ out_of_memory("prepare_sqlite");
+ if (DEBUG_GTE(DB, 3))
+ rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
+
+ while ((rc = sqlite3_prepare_v2(dbh.sqlite, query, -1, &statements[ndx].sqlite, NULL)) != 0) {
+ if (DEBUG_GTE(DB, 4)) {
+ rprintf(FCLIENT, "[%s] sqlite3_prepare_v2(,%s,,) returned %d\n",
+ who_am_i(), query, rc);
+ }
+ if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
+ break;
+ if (++lock_failures > MAX_LOCK_FAILURES)
+ break;
+ msleep(LOCK_FAIL_MSLEEP);
+ }
+ if (rc) {
+ rprintf(log_code, "[%s] Failed to prepare SQL: %s (%d)\n", who_am_i(), sqlite3_errmsg(dbh.sqlite), rc);
+ rprintf(log_code, "%s\n", query);
+ free(query);
+ return 0;
+ }
+ free(query);
+
+ return 1;
+}
+#endif
+
+#ifdef USE_SQLITE
+static int prepare_sqlite_queries(int type)
+{
+ char *sql;
+
+ switch (type) {
+ case PREP_NORM:
+ sql="SELECT disk_id"
+ " FROM disk"
+ " WHERE host = ? AND devno = ?";
+ if (!prepare_sqlite(SEL_DEV, sql))
+ return 0;
+
+ if (select_many_sums) {
+ sql="SELECT checksum, sum_type, size, mtime, ctime"
+ " FROM inode_map"
+ " WHERE disk_id = ? AND ino = ?";
+ if (!prepare_sqlite(SEL_SUM, sql))
+ return 0;
+ } else {
+ sql="SELECT checksum"
+ " FROM inode_map"
+ " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
+ " AND size = ? AND mtime = ? %s";
+ if (!prepare_sqlite(SEL_SUM, sql, md_num, db_lax ? "" : "AND ctime = ?"))
+ return 0;
+ }
+
+ sql="INSERT OR REPLACE INTO inode_map"
+ " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
+ " VALUES (?, ?, ?, ?, ?, ?, ?)";
+ if (!prepare_sqlite(REP_SUM, sql))
+ return 0;
+
+ sql="UPDATE inode_map"
+ " SET ctime = ?"
+ " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
+ if (!prepare_sqlite(UPD_CTIME, sql))
+ return 0;
+ break;
+
+ case PREP_MOUNT:
+ sql="INSERT OR IGNORE INTO disk"
+ " (host, last_seen, mount_uniq, devno)"
+ " VALUES (?, ?, ?, ?)";
+ if (!prepare_sqlite(INS_MOUNT, sql))
+ return 0;
+
+ sql="UPDATE disk"
+ " SET last_seen = ?, devno = ?"
+ " WHERE host = ? AND mount_uniq = ?";
+ if (!prepare_sqlite(UPD_MOUNT, sql))
+ return 0;
+
+ sql="SELECT mount_uniq"
+ " FROM disk"
+ " WHERE host = ? AND last_seen < ? AND devno != 0";
+ if (!prepare_sqlite(SEL_MOUNT, sql))
+ return 0;
+
+ sql="UPDATE disk"
+ " SET devno = 0"
+ " WHERE host = ? AND last_seen < ? AND devno != 0";
+ if (!prepare_sqlite(UN_MOUNT, sql))
+ return 0;
+ break;
+ }
+
+ return 1;
+}
+#endif
+
+#ifdef USE_SQLITE
+static int db_connect_sqlite(void)
+{
+ int lock_failures = 0;
+ int rc;
+
+#ifdef SQLITE_CONFIG_LOG
+ if (error_log) {
+ if (DEBUG_GTE(DB, 1))
+ rprintf(FCLIENT, "[%s] Setting sqlite errlog to %s\n", who_am_i(), error_log);
+ if (!(error_log_fp = fopen(error_log, "a"))) {
+ rsyserr(log_code, errno, "unable to append to logfile %s", error_log);
+ error_log = NULL;
+ } else if (sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, NULL) != 0)
+ rprintf(log_code, "Failed to set errorLogCallback: %s\n", sqlite3_errmsg(dbh.sqlite));
+ }
+#endif
+
+ while (1) {
+ int open_flags = SQLITE_OPEN_READWRITE;
+ if (db_init)
+ open_flags |= SQLITE_OPEN_CREATE;
+ if (DEBUG_GTE(DB, 1))
+ rprintf(FCLIENT, "[%s] opening %s (%d)\n", who_am_i(), dbname, open_flags);
+ if ((rc = sqlite3_open_v2(dbname, &dbh.sqlite, open_flags, NULL)) == 0) {
+ break;
+ }
+ if (DEBUG_GTE(DB, 4)) {
+ rprintf(FCLIENT, "[%s] sqlite3_open_v2(%s,,%d,NULL) returned %d\n",
+ who_am_i(), dbname, open_flags, rc);
+ }
+ if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
+ break;
+ if (++lock_failures > MAX_LOCK_FAILURES)
+ break;
+ msleep(LOCK_FAIL_MSLEEP);
+ }
+
+ if (rc) {
+ rprintf(log_code, "Unable to connect to DB: %s (%d)\n", sqlite3_errmsg(dbh.sqlite), rc);
+ return 0;
+ }
+
+ if (db_init) {
+ char *sql;
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Dropping old tables (if they exist) ...\n");
+ if (!run_sql("DROP TABLE IF EXISTS disk")
+ || !run_sql("DROP TABLE IF EXISTS inode_map"))
+ exit_cleanup(RERR_IPC);
+
+ if (db_output_msgs)
+ rprintf(FCLIENT, "Creating empty tables ...\n");
+ sql="CREATE TABLE disk (\n"
+ " disk_id integer NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
+ " host varchar(128) NOT NULL default 'localhost',\n"
+ " mount_uniq varchar(128) default NULL,\n"
+ " devno bigint NOT NULL,\n" /* This is 0 when not mounted */
+ " last_seen bigint NOT NULL,\n"
+ " UNIQUE (host, mount_uniq)\n"
+ ")";
+ if (!run_sql(sql))
+ exit_cleanup(RERR_IPC);
+
+ sql="CREATE TABLE inode_map (\n"
+ " disk_id integer NOT NULL,\n"
+ " ino bigint NOT NULL,\n"
+ " size bigint NOT NULL,\n"
+ " mtime bigint NOT NULL,\n"
+ " ctime bigint NOT NULL,\n"
+ " sum_type tinyint NOT NULL default '0',\n"
+ " checksum binary(16) NOT NULL,\n"
+ " PRIMARY KEY (disk_id,ino,sum_type)\n"
+ ")";
+ if (!run_sql(sql))
+ exit_cleanup(RERR_IPC);
+
+#if SQLITE_VERSION_NUMBER >= 3007000
+ /* Using WAL locking makes concurrency much better (requires sqlite 3.7.0). */
+ sql="PRAGMA journal_mode = wal";
+ run_sql(sql); /* We don't check this for success. */
+#endif
+
+ if (!db_mounts)
+ exit_cleanup(0);
+ }
+
+ if (db_mounts) {
+ if (!prepare_sqlite_queries(PREP_MOUNT))
+ exit_cleanup(RERR_IPC);
+ update_mounts();
+ exit_cleanup(0);
+ }
+
+ if (!prepare_sqlite_queries(PREP_NORM)) {
+ db_disconnect(False);
+ return 0;
+ }
+
+ return 1;
+}
+#endif
+
+int db_connect(int select_many)
+{
+ select_many_sums = select_many;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ if (db_connect_mysql())
+ return 1;
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE:
+ if (db_connect_sqlite())
+ return 1;
+ break;
+#endif
+ }
+
+ db_disconnect(False);
+
+ return 0;
+}
+
+void db_disconnect(BOOL commit)
+{
+ int ndx;
+
+ if (!dbh.all)
+ return;
+
+ if (transaction_state > 0) {
+ if (DEBUG_GTE(DB, 1)) {
+ rprintf(FCLIENT, "[%s] %s our DB transaction\n",
+ who_am_i(), commit ? "Committing" : "Rolling back");
+ }
+ transaction_state = 0;
+ if (commit)
+ run_sql("COMMIT");
+ else
+ run_sql("ROLLBACK");
+ }
+
+ if (DEBUG_GTE(DB, 1))
+ rprintf(FCLIENT, "[%s] Disconnecting from the DB\n", who_am_i());
+
+ for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
+ if (statements[ndx].all) {
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ mysql_stmt_close(statements[ndx].mysql);
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE:
+ sqlite3_finalize(statements[ndx].sqlite);
+ break;
+#endif
+ }
+ statements[ndx].all = NULL;
+ }
+ }
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ mysql_close(dbh.mysql);
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE:
+ sqlite3_close(dbh.sqlite);
+ break;
+#endif
+ }
+
+ dbh.all = NULL;
+ use_db = DB_TYPE_NONE;
+}
+
+#ifdef USE_MYSQL
+static MYSQL_STMT *exec_mysql(int ndx)
+{
+ MYSQL_STMT *stmt = statements[ndx].mysql;
+ int rc;
+
+ if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
+ db_disconnect(False);
+ use_db = DB_TYPE_MYSQL;
+ if (db_connect(select_many_sums)) {
+ stmt = statements[ndx].mysql;
+ rc = mysql_stmt_execute(stmt);
+ }
+ }
+ if (rc != 0) {
+ rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
+ return NULL;
+ }
+
+ return stmt;
+}
+#endif
+
+#ifdef USE_MYSQL
+/* This stores up to max_rows into the values pointed to by the bind data arrays.
+ * If max_rows is > 1, then all the buffer pointers MUST be set to an array long
+ * enough to hold the max count of rows. The buffer pointer will be incremented
+ * to read additional rows (but never past the end). If stmt_ptr is non-NULL, it
+ * will be set to the "stmt" pointer IFF we didn't run out of rows before hitting
+ * the max. In this case, the caller should call mysql_stmt_fetch() to read any
+ * remaining rows (the buffer pointers will point at the final array element) and
+ * then call mysql_stmt_free_result(). If *stmt_ptr is a NULL value, there were
+ * not enough rows to fill the max_rows arrays, and the stmt was already freed. */
+static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx, int max_rows, MYSQL_STMT **stmt_ptr)
+{
+ MYSQL_STMT *stmt;
+ int i, rc, rows = 0;
+
+ if (bind_cnt > MAX_RESULT_BINDS) {
+ fprintf(stderr, "Internal error: MAX_RESULT_BINDS overflow\n");
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+
+ if ((stmt = exec_mysql(ndx)) == NULL)
+ return 0;
+
+ for (i = 0; i < bind_cnt; i++) {
+ binds[i].is_null = &result_is_null[i];
+ binds[i].length = &result_length[i];
+ binds[i].error = &result_error[i];
+ }
+ mysql_stmt_bind_result(stmt, binds);
+
+ while (rows < max_rows) {
+ if ((rc = mysql_stmt_fetch(stmt)) != 0) {
+ if (rc != MYSQL_NO_DATA)
+ rprintf(log_code, "SELECT fetch failed: %s\n", mysql_stmt_error(stmt));
+ break;
+ }
+ if (++rows >= max_rows)
+ break;
+ for (i = 0; i < bind_cnt; i++) {
+ switch (binds[i].buffer_type) {
+ case MYSQL_TYPE_BLOB:
+ case MYSQL_TYPE_STRING:
+ binds[i].buffer += binds[i].buffer_length;
+ break;
+ case MYSQL_TYPE_LONG:
+ binds[i].buffer += sizeof (int);
+ break;
+ case MYSQL_TYPE_LONGLONG:
+ binds[i].buffer += sizeof (int64);
+ break;
+ default:
+ fprintf(stderr, "Unknown MYSQL_TYPE_* in multi-row read: %d.\n", binds[i].buffer_type);
+ exit_cleanup(RERR_UNSUPPORTED);
+ }
+ }
+ }
+
+ if (!stmt_ptr || rows < max_rows) {
+ mysql_stmt_free_result(stmt);
+ stmt = NULL;
+ }
+ if (stmt_ptr)
+ *stmt_ptr = stmt;
+
+ return rows;
+}
+#endif
+
+#if defined USE_MYSQL || defined USE_SQLITE
+static void update_mounts(void)
+{
+ char buf[2048], *argv[2];
+ int f_from, f_to, len;
+ STRUCT_STAT st;
+ int pid, status;
+
+ if (DEBUG_GTE(DB, 2))
+ printf("Running %s to grab mount info\n", mount_helper_script);
+ argv[0] = mount_helper_script;
+ argv[1] = NULL;
+ pid = piped_child(argv, &f_from, &f_to);
+ close(f_to);
+
+ bind_mtime = time(NULL); /* abuse mtime slightly to hold our last_seen value */
+
+ /* Strict format has 2 items with one tab as separator: MOUNT_UNIQ\tPATH */
+ while ((len = read_line(f_from, buf, sizeof buf, 0)) > 0) {
+ char *mount_uniq, *path;
+
+ if (DEBUG_GTE(DB, 3))
+ printf("Parsing mount info: %s\n", buf);
+ mount_uniq = strtok(buf, "\t");
+ path = mount_uniq ? strtok(NULL, "\r\n") : NULL;
+ if (!path) {
+ fprintf(stderr, "Failed to parse line from %s output\n", mount_helper_script);
+ exit_cleanup(RERR_SYNTAX);
+ }
+
+ if (lstat(path, &st) < 0) {
+ fprintf(stderr, "Failed to lstat(%s): %s\n", path, strerror(errno));
+ exit_cleanup(RERR_IPC);
+ }
+
+ bind_mount_uniq_len = strlcpy(bind_mount_uniq, mount_uniq, sizeof bind_mount_uniq);
+ if (bind_mount_uniq_len >= (int)sizeof bind_mount_uniq)
+ bind_mount_uniq_len = sizeof bind_mount_uniq - 1;
+
+ if (db_output_msgs) {
+ printf("Marking mount \"%s\" (%s) as a recent mount\n",
+ bind_mount_uniq, big_num(st.st_dev));
+ }
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ bind_devno = st.st_dev;
+ if (exec_mysql(INS_MOUNT) == NULL) {
+ fprintf(stderr, "Failed to update mount info for \"%s\" - %s\n",
+ bind_mount_uniq, mysql_error(dbh.mysql));
+ exit_cleanup(RERR_IPC);
+ }
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc, change_cnt;
+ sqlite3_stmt *stmt = statements[INS_MOUNT].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 2, bind_mtime);
+ sqlite3_bind_text(stmt, 3, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 4, st.st_dev);
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_DONE) {
+ fprintf(stderr, "Failed to insert mount info for \"%s\" - %s (%d)\n",
+ bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
+ exit_cleanup(RERR_IPC);
+ }
+ change_cnt = sqlite3_changes(dbh.sqlite);
+ sqlite3_reset(stmt);
+ if (change_cnt == 0) {
+ stmt = statements[UPD_MOUNT].sqlite;
+ sqlite3_bind_int64(stmt, 1, bind_mtime);
+ sqlite3_bind_int64(stmt, 2, st.st_dev);
+ sqlite3_bind_text(stmt, 3, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_text(stmt, 4, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_DONE) {
+ fprintf(stderr, "Failed to update mount info for \"%s\" - %s (%d)\n",
+ bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
+ exit_cleanup(RERR_IPC);
+ }
+ sqlite3_reset(stmt);
+ }
+ break;
+ }
+#endif
+ }
+ }
+ close(f_from);
+
+ waitpid(pid, &status, 0);
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ if (db_output_msgs) {
+ MYSQL_BIND binds[1];
+ MYSQL_STMT *stmt;
+
+ binds[0].buffer_type = MYSQL_TYPE_BLOB;
+ binds[0].buffer = bind_mount_uniq;
+ binds[0].buffer_length = sizeof bind_mount_uniq;
+ if (fetch_mysql(binds, 1, SEL_MOUNT, 1, &stmt)) {
+ while (1) {
+ printf("Marking mount \"%s\" as unmounted.\n", bind_mount_uniq);
+ if (mysql_stmt_fetch(stmt) != 0)
+ break;
+ }
+ mysql_stmt_free_result(stmt);
+ }
+ }
+
+ if (exec_mysql(UN_MOUNT) == NULL) {
+ fprintf(stderr, "Failed to update old mount info - %s\n",
+ mysql_error(dbh.mysql));
+ exit_cleanup(RERR_IPC);
+ }
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt;
+ int rc;
+
+ if (db_output_msgs) {
+ stmt = statements[SEL_MOUNT].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 2, bind_mtime);
+ while (1) {
+ if (sqlite3_step(stmt) != SQLITE_ROW)
+ break;
+ printf("Marking mount \"%s\" as unmounted.\n", sqlite3_column_text(stmt, 0));
+ }
+ sqlite3_reset(stmt);
+ }
+
+ stmt = statements[UN_MOUNT].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 2, bind_mtime);
+ rc = sqlite3_step(stmt);
+ sqlite3_reset(stmt);
+ if (rc != SQLITE_DONE) {
+ fprintf(stderr, "Failed to update old mount info - %s (%d)\n",
+ sqlite3_errmsg(dbh.sqlite), rc);
+ exit_cleanup(RERR_IPC);
+ }
+ break;
+ }
+#endif
+ }
+}
+#endif
+
+static unsigned int get_disk_id(int64 devno)
+{
+ static unsigned int prior_disk_id = 0;
+ static int64 prior_devno = 0;
+
+ if (prior_devno == devno && prior_disk_id) {
+ if (DEBUG_GTE(DB, 5))
+ rprintf(FCLIENT, "get_disk_id(%s,%s) = %d (cached)\n", bind_thishost, big_num(devno), prior_disk_id);
+ return prior_disk_id;
+ }
+ prior_devno = devno;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_BIND binds[1];
+
+ bind_devno = devno; /* The one changing SEL_DEV input value. */
+
+ /* Bind where to put the output. */
+ binds[0].buffer_type = MYSQL_TYPE_LONG;
+ binds[0].buffer = &prior_disk_id;
+ if (!fetch_mysql(binds, 1, SEL_DEV, 1, NULL))
+ prior_disk_id = 0;
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 2, devno);
+ if (sqlite3_step(stmt) == SQLITE_ROW)
+ prior_disk_id = sqlite3_column_int(stmt, 0);
+ else
+ prior_disk_id = 0;
+ sqlite3_reset(stmt);
+ break;
+ }
+#endif
+ }
+
+ if (DEBUG_GTE(DB, 2))
+ rprintf(FCLIENT, "get_disk_id(%s,%s) = %d\n", bind_thishost, big_num(devno), prior_disk_id);
+ return prior_disk_id;
+}
+
+int db_get_checksum(const STRUCT_STAT *st_p, char *sum)
+{
+ unsigned int disk_id = get_disk_id(st_p->st_dev);
+ int ok = 0;
+
+ if (disk_id == 0)
+ return 0;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_BIND binds[1];
+
+ bind_disk_id = disk_id;
+ bind_ino = st_p->st_ino;
+ bind_size = st_p->st_size;
+ bind_mtime = st_p->st_mtime;
+ if (!db_lax)
+ bind_ctime = st_p->st_ctime;
+
+ binds[0].buffer_type = MYSQL_TYPE_BLOB;
+ binds[0].buffer = sum;
+ binds[0].buffer_length = MD5_DIGEST_LEN;
+ ok = fetch_mysql(binds, 1, SEL_SUM, 1, NULL);
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
+ sqlite3_bind_int(stmt, 1, disk_id);
+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
+ sqlite3_bind_int64(stmt, 3, st_p->st_size);
+ sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
+ if (!db_lax)
+ sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
+ if (sqlite3_step(stmt) == SQLITE_ROW) {
+ int len = sqlite3_column_bytes(stmt, 0);
+ if (len > MAX_DIGEST_LEN)
+ len = MAX_DIGEST_LEN;
+ memcpy(sum, sqlite3_column_blob(stmt, 0), len);
+ ok = 1;
+ }
+ sqlite3_reset(stmt);
+ break;
+ }
+#endif
+ }
+
+ if (DEBUG_GTE(DB, 2)) {
+ if (ok) {
+ rprintf(FCLIENT, "[%s] Found DB checksum for %s,%s,%d: %s\n",
+ who_am_i(), big_num(st_p->st_dev),
+ big_num(st_p->st_ino), md_num, sum_as_hex(md_num, sum, 0));
+ } else {
+ rprintf(FCLIENT, "[%s] No DB checksum for %s,%s,%d\n",
+ who_am_i(), big_num(st_p->st_dev),
+ big_num(st_p->st_ino), md_num);
+ }
+ }
+
+ return ok;
+}
+
+int db_get_both_checksums(const STRUCT_STAT *st_p, int *right_sum_cnt, int *wrong_sum_cnt, char **sum4, char **sum5)
+{
+ static char dbsum[MD5_DIGEST_LEN*2];
+ int rows, j, sum_type[2];
+ int64 dbsize[2], dbmtime[2], dbctime[2];
+ unsigned int disk_id = get_disk_id(st_p->st_dev);
+
+ if (disk_id == 0)
+ return 0;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_BIND binds[5];
+
+ bind_disk_id = disk_id;
+ bind_ino = st_p->st_ino;
+
+ binds[0].buffer_type = MYSQL_TYPE_BLOB;
+ binds[0].buffer = dbsum;
+ binds[0].buffer_length = MD5_DIGEST_LEN;
+ binds[1].buffer_type = MYSQL_TYPE_LONG;
+ binds[1].buffer = (char*)sum_type;
+ binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[2].buffer = (char*)dbsize;
+ binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[3].buffer = (char*)dbmtime;
+ binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[4].buffer = (char*)dbctime;
+ rows = fetch_mysql(binds, 5, SEL_SUM, 2, NULL);
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
+ sqlite3_bind_int(stmt, 1, disk_id);
+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
+ for (j = 0; j < 2; j++) {
+ int len;
+ if (sqlite3_step(stmt) != SQLITE_ROW)
+ break;
+ len = sqlite3_column_bytes(stmt, 0);
+ if (len > MD5_DIGEST_LEN)
+ len = MD5_DIGEST_LEN;
+ memcpy(dbsum + MD5_DIGEST_LEN*j, sqlite3_column_blob(stmt, 0), len);
+ sum_type[j] = sqlite3_column_int(stmt, 1);
+ dbsize[j] = sqlite3_column_int(stmt, 2);
+ dbmtime[j] = sqlite3_column_int64(stmt, 3);
+ dbctime[j] = sqlite3_column_int64(stmt, 4);
+ }
+ sqlite3_reset(stmt);
+ rows = j;
+ break;
+ }
+#endif
+ default:
+ return 0;
+ }
+
+ if (sum4)
+ *sum4 = NULL;
+ if (sum5)
+ *sum5 = NULL;
+ *right_sum_cnt = *wrong_sum_cnt = 0;
+ for (j = 0; j < rows; j++) {
+ if (DEBUG_GTE(DB, 3)) {
+ rprintf(FCLIENT, "DB checksum for %s,%s,%d: %s\n",
+ big_num(st_p->st_dev), big_num(st_p->st_ino), sum_type[j],
+ sum_as_hex(sum_type[j], dbsum + MD5_DIGEST_LEN*j, 0));
+ }
+
+ if (sum_type[j] == 4) {
+ if (!sum4)
+ continue;
+ *sum4 = dbsum + MD5_DIGEST_LEN*j;
+ } else {
+ if (!sum5)
+ continue;
+ *sum5 = dbsum + MD5_DIGEST_LEN*j;
+ }
+ if (st_p->st_size == dbsize[j] && st_p->st_mtime == dbmtime[j] && (db_lax || st_p->st_ctime == dbctime[j]))
+ ++*right_sum_cnt;
+ else
+ ++*wrong_sum_cnt;
+ }
+
+ return rows;
+}
+
+int db_set_checksum(int mdnum, const STRUCT_STAT *st_p, const char *sum)
+{
+ unsigned int disk_id;
+ const char *errmsg = NULL;
+ int rc = 0;
+
+ if (am_receiver || (am_generator && same_db)) {
+ /* Forward the setting to a single process. The receiver always
+ * forwards to the generator, and the generator will forward to
+ * the sender ONLY if this is a local transfer. */
+ char data[MSG_CHECKSUM_LEN];
+ SIVAL64(data, 0, st_p->st_dev);
+ SIVAL64(data, 8, st_p->st_ino);
+ SIVAL64(data, 16, st_p->st_size);
+ SIVAL64(data, 24, st_p->st_mtime);
+ SIVAL64(data, 32, st_p->st_ctime);
+#if MSG_CHECKSUM_LONGS != 5
+#error Fix the setting of checksum long values
+#endif
+ SIVAL(data, MSG_CHECKSUM_LONGS*8, mdnum);
+ memcpy(data + MSG_CHECKSUM_LONGS*8 + 4, sum, MAX_DIGEST_LEN);
+ return send_msg(MSG_CHECKSUM, data, sizeof data, 0);
+ }
+
+ if ((disk_id = get_disk_id(st_p->st_dev)) == 0)
+ return 0;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ if (transaction_state == 0) {
+ if (!run_sql("BEGIN"))
+ return 0;
+ transaction_state = 1;
+ }
+
+ bind_disk_id = disk_id;
+ bind_ino = st_p->st_ino;
+ bind_mdnum = mdnum;
+ bind_size = st_p->st_size;
+ bind_mtime = st_p->st_mtime;
+ bind_ctime = st_p->st_ctime;
+ memcpy(bind_sum, sum, MD5_DIGEST_LEN);
+ if (exec_mysql(REP_SUM) == NULL)
+ errmsg = mysql_error(dbh.mysql);
+ break;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
+ int lock_failures = 0;
+
+ if (transaction_state == 0) {
+ if (!run_sql("BEGIN"))
+ return 0;
+ transaction_state = 1;
+ }
+
+ sqlite3_bind_int(stmt, 1, disk_id);
+ sqlite3_bind_int64(stmt, 2, st_p->st_ino);
+ sqlite3_bind_int(stmt, 3, mdnum);
+ sqlite3_bind_int64(stmt, 4, st_p->st_size);
+ sqlite3_bind_int64(stmt, 5, st_p->st_mtime);
+ sqlite3_bind_int64(stmt, 6, st_p->st_ctime);
+ sqlite3_bind_blob(stmt, 7, sum, MD5_DIGEST_LEN, SQLITE_TRANSIENT);
+ while (1) {
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
+ break;
+ if (++lock_failures > MAX_LOCK_FAILURES)
+ break;
+ sqlite3_reset(stmt);
+ msleep(LOCK_FAIL_MSLEEP);
+ }
+ if (rc != SQLITE_DONE)
+ errmsg = sqlite3_errmsg(dbh.sqlite);
+ sqlite3_reset(stmt);
+ break;
+ }
+#endif
+ }
+
+ if (!errmsg) {
+ if (DEBUG_GTE(DB, 2)) {
+ rprintf(FCLIENT, "[%s] Set DB checksum for %s,%s,%d: %s\n",
+ who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
+ md_num, sum_as_hex(md_num, sum, 0));
+ }
+ } else {
+ rprintf(log_code, "[%s] Failed to set checksum for %s,%s,%d: %s (%d) -- closing DB\n",
+ who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
+ md_num, errmsg, rc);
+ db_disconnect(False);
+ }
+
+ return errmsg ? 0 : 1;
+}
+
+/* For a delayed-update copy, we set the checksum on the file when it was
+ * inside the partial-dir. Since renaming the file changes its ctime, we need
+ * to update the ctime to its new value (we can skip this in db_lax mode). */
+int db_update_ctime(UNUSED(int mdnum), const STRUCT_STAT *st_p)
+{
+ unsigned int disk_id = get_disk_id(st_p->st_dev);
+
+ if (disk_id == 0)
+ return 0;
+
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ bind_ctime = st_p->st_ctime;
+ bind_disk_id = disk_id;
+ bind_ino = st_p->st_ino;
+ bind_mdnum = mdnum;
+ bind_size = st_p->st_size;
+ bind_mtime = st_p->st_mtime;
+ return exec_mysql(UPD_CTIME) != NULL;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc;
+
+ sqlite3_stmt *stmt = statements[UPD_CTIME].sqlite;
+ if (stmt == NULL)
+ return 0;
+ sqlite3_bind_int64(stmt, 1, st_p->st_ctime);
+ sqlite3_bind_int(stmt, 2, disk_id);
+ sqlite3_bind_int64(stmt, 3, st_p->st_ino);
+ sqlite3_bind_int(stmt, 4, mdnum);
+ sqlite3_bind_int64(stmt, 5, st_p->st_size);
+ sqlite3_bind_int64(stmt, 6, st_p->st_mtime);
+ rc = sqlite3_step(stmt);
+ sqlite3_reset(stmt);
+ return rc == SQLITE_DONE;
+ }
+#endif
+ }
+
+ return 0;
+}
+
+static int db_clean_init(void)
+{
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_BIND binds[MAX_BIND_CNT];
+ char *sql;
+
+ mysql_query(dbh.mysql,
+ "CREATE TEMPORARY TABLE inode_present ("
+ " disk_id integer unsigned NOT NULL,"
+ " ino bigint unsigned NOT NULL,"
+ " PRIMARY KEY (disk_id,ino)"
+ ") ENGINE=MEMORY"
+ );
+
+ sql="INSERT IGNORE INTO inode_present"
+ " SET disk_id = ?, ino = ?";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_LONG;
+ binds[0].buffer = &bind_disk_id;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_ino;
+ if (!prepare_mysql(INS_PRESENT, binds, 2, sql))
+ exit_cleanup(RERR_SYNTAX);
+
+ sql="DELETE m.*"
+ " FROM inode_map AS m"
+ " LEFT JOIN inode_present AS p USING(disk_id, ino)"
+ " JOIN disk AS d ON(m.disk_id = d.disk_id)"
+ " WHERE host = ? AND devno != 0 AND p.disk_id IS NULL AND ctime < ?";
+ memset(binds, 0, sizeof binds);
+ binds[0].buffer_type = MYSQL_TYPE_STRING;
+ binds[0].buffer = &bind_thishost;
+ binds[0].buffer_length = bind_thishost_len;
+ binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
+ binds[1].buffer = &bind_ctime;
+ if (!prepare_mysql(DEL_SUMS, binds, 2, sql))
+ exit_cleanup(RERR_SYNTAX);
+
+ return 1;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ char *sql;
+ sql="ATTACH DATABASE '' AS aux1;"; /* Private temp DB, probably in-memory */
+ if (!run_sql(sql))
+ exit_cleanup(RERR_IPC);
+
+ sql="CREATE TABLE aux1.inode_present ("
+ " disk_id integer NOT NULL,"
+ " ino bigint NOT NULL,"
+ " PRIMARY KEY (disk_id,ino)"
+ ")";
+ if (!run_sql(sql))
+ exit_cleanup(RERR_IPC);
+
+ sql="INSERT OR IGNORE INTO aux1.inode_present"
+ " (disk_id, ino)"
+ " VALUES (?, ?)";
+ if (!prepare_sqlite(INS_PRESENT, sql))
+ exit_cleanup(RERR_IPC);
+
+ sql="DELETE FROM inode_map"
+ " WHERE ROWID IN ("
+ " SELECT m.ROWID"
+ " FROM inode_map AS m"
+ " LEFT JOIN aux1.inode_present AS p USING(disk_id, ino)"
+ " JOIN disk AS d ON(m.disk_id = d.disk_id)"
+ " WHERE host = ? AND devno != 0 AND p.disk_id IS NULL AND ctime < ?"
+ " )";
+ if (!prepare_sqlite(DEL_SUMS, sql))
+ exit_cleanup(RERR_IPC);
+
+ transaction_state = -1; /* bug work-around -- force transaction off when cleaning XXX */
+
+ return 1;
+ }
+#endif
+ }
+
+ return 0;
+}
+
+static int db_note_present(UNUSED(int disk_id), UNUSED(int64 ino))
+{
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL:
+ bind_disk_id = disk_id;
+ bind_ino = ino;
+ return exec_mysql(INS_PRESENT) != NULL;
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc;
+ sqlite3_stmt *stmt = statements[INS_PRESENT].sqlite;
+ sqlite3_bind_int(stmt, 1, disk_id);
+ sqlite3_bind_int64(stmt, 2, ino);
+ rc = sqlite3_step(stmt);
+ sqlite3_reset(stmt);
+ return rc == SQLITE_DONE;
+ }
+#endif
+ }
+
+ return 0;
+}
+
+/* This function requires the user to have populated all disk_id+inode pairs
+ * into the inode_present table. */
+static int db_clean_inodes(UNUSED(time_t start_time))
+{
+ int del_cnt = 0;
+
+ /* The extra ctime < start_time check ensures that brand-new checksums that
+ * were added after the start of our cleaning run are not removed. */
+ switch (use_db) {
+#ifdef USE_MYSQL
+ case DB_TYPE_MYSQL: {
+ MYSQL_STMT *stmt;
+ bind_ctime = start_time;
+ stmt = exec_mysql(DEL_SUMS);
+ if (stmt != NULL)
+ del_cnt = mysql_affected_rows(dbh.mysql);
+ break;
+ }
+#endif
+#ifdef USE_SQLITE
+ case DB_TYPE_SQLITE: {
+ int rc;
+ sqlite3_stmt *stmt = statements[DEL_SUMS].sqlite;
+ sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
+ sqlite3_bind_int64(stmt, 2, start_time);
+ rc = sqlite3_step(stmt);
+ if (rc == SQLITE_DONE)
+ del_cnt = sqlite3_changes(dbh.sqlite);
+ sqlite3_reset(stmt);
+ break;
+ }
+#endif
+ }
+
+ return del_cnt;
+}
+
+static int abs_path(char *buf, int bufsiz, const char *curdir, const char *dir)
+{
+ if (*dir == '/')
+ strlcpy(buf, dir, bufsiz);
+ else {
+ int len = snprintf(buf, bufsiz, "%s/%s", curdir, dir);
+ assert(len > 0); /* silence a compiler warning */
+ }
+
+ return clean_fname(buf, CFN_DROP_TRAILING_DOT_DIR | CFN_COLLAPSE_DOT_DOT_DIRS);
+}
+
+static struct name_list *new_name(const char *basename, const char *filename)
+{
+ struct name_list *n;
+ int blen = strlen(basename);
+ int slen = filename ? (int)strlen(filename) : -1;
+ int len = blen + 1 + slen;
+
+ if (len >= MAXPATHLEN) {
+ if (filename)
+ rprintf(FERROR, "Filename too long: %s/%s\n", basename, filename);
+ else
+ rprintf(FERROR, "Filename too long: %s\n", basename);
+ return NULL;
+ }
+
+ n = (struct name_list *)new_array(char, sizeof (struct name_list) + len);
+
+ memcpy(n->name, basename, blen);
+ if (filename) {
+ n->name[blen] = '/';
+ memcpy(n->name + 1 + blen, filename, slen);
+ }
+ n->name[len] = '\0';
+ n->next = NULL;
+
+ return n;
+}
+
+static int name_compare(const void *n1, const void *n2)
+{
+ struct name_list *p1 = *(struct name_list **)n1;
+ struct name_list *p2 = *(struct name_list **)n2;
+ return strcmp(p1->name, p2->name);
+}
+
+static struct name_list *get_sorted_names(const char *dir)
+{
+ struct name_list *add, **sortbuf, *names = NULL, *prior_name = NULL;
+ struct dirent *di;
+ int cnt = 0;
+ DIR *d;
+
+ if (!(d = opendir("."))) {
+ rprintf(FERROR, "Unable to opendir %s: %s\n", dir, strerror(errno));
+ return NULL;
+ }
+ while ((di = readdir(d)) != NULL) {
+ char *dname = d_name(di);
+ if (dname[0] == '.' && (dname[1] == '\0' || (dname[1] == '.' && dname[2] == '\0')))
+ continue;
+ if (!(add = new_name(dname, NULL)))
+ continue;
+ if (prior_name)
+ prior_name->next = add;
+ else
+ names = add;
+ prior_name = add;
+ cnt++;
+ }
+ closedir(d);
+
+ if (cnt) {
+ int j;
+
+ sortbuf = new_array(struct name_list *, cnt);
+ for (j = 0; j < cnt; j++) {
+ sortbuf[j] = names;
+ names = names->next;
+ }
+
+ qsort(sortbuf, cnt, PTR_SIZE, name_compare);
+
+ names = prior_name = NULL;
+ for (j = 0; j < cnt; j++) {
+ add = sortbuf[j];
+ if (prior_name)
+ prior_name->next = add;
+ else
+ names = add;
+ prior_name = add;
+ }
+
+ if (prior_name)
+ prior_name->next = NULL;
+ free(sortbuf);
+ }
+
+ return names;
+}
+
+static inline int sums_ne(const char *sum1, const char *sum2)
+{
+ return memcmp(sum1, sum2, MD5_DIGEST_LEN) != 0;
+}
+
+/* Returns 1 if there is a checksum change, else 0. */
+static int mention_file(const char *dir, const char *name, int right_cnt, int wrong_cnt,
+ const char *dbsum4, const char *dbsum5, const char *sum4, const char *sum5)
+{
+ char *info_str = wrong_cnt && !right_cnt ? "!i " : " ";
+ char *md4_str = !db_do_md4 ? NULL : !dbsum4 ? "+4 " : !sum4 ? "?4 " : sums_ne(sum4, dbsum4) ? "!4 " : " ";
+ char *md5_str = !db_do_md5 ? NULL : !dbsum5 ? "+5 " : !sum5 ? "?5 " : sums_ne(sum5, dbsum5) ? "!5 " : " ";
+ int chg = *info_str != ' ' || (md4_str && *md4_str != ' ') || (md5_str && *md5_str != ' ');
+ if (chg || db_output_unchanged) {
+ if (db_output_info) {
+ fputs(info_str, stdout);
+ if (md4_str)
+ fputs(md4_str, stdout);
+ if (md5_str)
+ fputs(md5_str, stdout);
+ }
+ if (db_output_sum) {
+ if (db_do_md4)
+ printf("%s ", sum_as_hex(4, sum4, 0));
+ if (db_do_md5)
+ printf("%s ", sum_as_hex(5, sum5, 0));
+ }
+ if (db_output_name) {
+ if (db_output_sum)
+ putchar(' '); /* We want 2 spaces, like md5sum. */
+ if (*dir != '.' || dir[1]) {
+ fputs(dir, stdout);
+ putchar('/');
+ }
+ puts(name);
+ }
+ }
+
+ return chg;
+}
+
+NORETURN void run_dbonly(const char **args)
+{
+ char start_dir[MAXPATHLEN], dirbuf[MAXPATHLEN];
+ int need_sum_cnt, start_dir_len;
+ struct name_list *prior_dir;
+ struct name_list *names;
+ time_t clean_start = 0;
+ int exit_code = 0;
+
+ checksum_type = 5;
+
+ need_sum_cnt = db_do_md4 + db_do_md5;
+
+ if (!db_read_config(FERROR, db_config) || !db_connect(1))
+ exit_cleanup(RERR_FILEIO);
+
+ if (db_clean) {
+ clean_start = time(NULL);
+ db_clean_init();
+ }
+
+ if (getcwd(start_dir, sizeof start_dir - 1) == NULL) {
+ rsyserr(FERROR, errno, "getcwd()");
+ exit_cleanup(RERR_FILESELECT);
+ }
+ start_dir_len = strlen(start_dir);
+
+ if (args) {
+ prior_dir = NULL;
+ while (*args) {
+ struct name_list *add;
+ if (abs_path(dirbuf, sizeof dirbuf, start_dir, *args++) <= 0)
+ continue;
+ if (!(add = new_name(dirbuf, NULL)))
+ continue;
+ if (prior_dir)
+ prior_dir->next = add;
+ else
+ dirs_list = add;
+ prior_dir = add;
+ }
+ } else
+ dirs_list = new_name(start_dir, NULL);
+
+ prior_dir = NULL;
+ while (dirs_list) {
+ struct name_list *subdirs, *prior_subdir, *prior_name;
+ const char *dir = dirs_list->name;
+ const char *reldir = dir;
+
+ if (prior_dir)
+ free((void*)prior_dir);
+ prior_dir = dirs_list;
+ dirs_list = dirs_list->next;
+
+ if (strncmp(reldir, start_dir, start_dir_len) == 0) {
+ if (reldir[start_dir_len] == '\0')
+ reldir = ".";
+ else if (reldir[start_dir_len] == '/')
+ reldir += start_dir_len + 1;
+ }
+ if (db_output_dirs)
+ printf("... %s/ ...\n", reldir);
+
+ if (chdir(dir) < 0) {
+ rprintf(FERROR, "Unable to chdir to %s: %s\n", dir, strerror(errno));
+ continue;
+ }
+ if (!(names = get_sorted_names(dir)))
+ continue;
+
+ subdirs = prior_subdir = prior_name = NULL;
+ while (names) {
+ STRUCT_STAT st;
+ char *dbsum4, *sum4, sumbuf4[MD5_DIGEST_LEN];
+ char *dbsum5, *sum5, sumbuf5[MD5_DIGEST_LEN];
+ int right_sum_cnt, wrong_sum_cnt;
+ const char *name = names->name;
+ unsigned int disk_id;
+
+ if (prior_name)
+ free((void*)prior_name);
+ prior_name = names;
+ names = names->next;
+
+ dbsum4 = dbsum5 = sum4 = sum5 = NULL;
+
+ if (lstat(name, &st) < 0) {
+ rprintf(FERROR, "Failed to lstat(%s): %s\n", name, strerror(errno));
+ continue;
+ }
+ if (S_ISLNK(st.st_mode))
+ continue;
+ if (S_ISDIR(st.st_mode)) {
+ /* add optional excluding of things like /^(CVS|\.svn|\.git|\.bzr)$/; */
+ if (recurse) {
+ struct name_list *add = new_name(dir, name);
+ if (add) {
+ if (prior_subdir)
+ prior_subdir->next = add;
+ else
+ subdirs = add;
+ prior_subdir = add;
+ }
+ }
+ continue;
+ }
+ if (!S_ISREG(st.st_mode))
+ continue;
+
+ if (!(disk_id = get_disk_id(st.st_dev)))
+ continue;
+ if (db_clean) {
+ db_note_present(disk_id, st.st_ino);
+ if (!db_update && !db_check)
+ continue;
+ }
+ db_get_both_checksums(&st, &right_sum_cnt, &wrong_sum_cnt,
+ db_do_md4 ? &dbsum4 : NULL, db_do_md5 ? &dbsum5 : NULL);
+
+ if (!db_check && right_sum_cnt == need_sum_cnt) {
+ mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, dbsum4, dbsum5);
+ continue;
+ }
+
+ if (db_update || (db_check && right_sum_cnt) || db_output_sum) {
+ uchar *data;
+ int32 remainder;
+ md_context m4;
+ MD5_CTX m5;
+ struct map_struct *buf;
+ OFF_T off, len = st.st_size;
+ int fd = do_open(name, O_RDONLY, 0);
+
+ if (fd < 0) {
+ rprintf(FERROR, "ERROR: unable to read %s: %s\n", name, strerror(errno));
+ continue;
+ }
+
+ if (db_do_md4)
+ mdfour_begin(&m4);
+ if (db_do_md5)
+ MD5_Init(&m5);
+
+ buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
+
+ for (off = 0; off + CSUM_CHUNK <= len; off += CSUM_CHUNK) {
+ data = (uchar*)map_ptr(buf, off, CSUM_CHUNK);
+ if (db_do_md4)
+ mdfour_update(&m4, data, CSUM_CHUNK);
+ if (db_do_md5)
+ MD5_Update(&m5, data, CSUM_CHUNK);
+ }
+
+ remainder = (int32)(len - off);
+ data = (uchar*)map_ptr(buf, off, remainder);
+ if (db_do_md4) {
+ mdfour_update(&m4, data, remainder);
+ mdfour_result(&m4, (uchar*)(sum4 = sumbuf4));
+ }
+ if (db_do_md5) {
+ MD5_Update(&m5, data, remainder);
+ MD5_Final((uchar*)(sum5 = sumbuf5), &m5);
+ }
+
+ close(fd);
+ unmap_file(buf);
+ }
+
+ int chg = mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, sum4, sum5);
+ if (!chg) {
+ /* Only db_check should get here... */
+ } else if (!db_update) {
+ exit_code = 1;
+ } else {
+ int fail = 0;
+ if (db_do_md4 && !db_set_checksum(4, &st, sum4))
+ fail = 1;
+ if (db_do_md5 && !db_set_checksum(5, &st, sum5))
+ fail = 1;
+ if (fail) {
+ fprintf(stderr, "Failed to set checksum on %s/%s\n", reldir, name);
+ exit_cleanup(RERR_FILEIO);
+ }
+ }
+ }
+ if (prior_name)
+ free((void*)prior_name);
+
+ if (recurse && subdirs) {
+ prior_subdir->next = dirs_list;
+ dirs_list = subdirs;
+ }
+ }
+ if (prior_dir)
+ free((void*)prior_dir);
+
+ if (db_clean) {
+ int rows = db_clean_inodes(clean_start);
+ if (db_output_msgs)
+ printf("Cleaned out %d old inode%s.\n", rows, rows == 1 ? "" : "s");
+ }
+
+ db_disconnect(True);
+ exit(exit_code);
+}
diff --git a/flist.c b/flist.c
--- a/flist.c
+++ b/flist.c
@@ -54,6 +54,7 @@ extern int preserve_devices;
extern int preserve_specials;
extern int delete_during;
extern int missing_args;
+extern int use_db;
extern int eol_nulls;
extern int atimes_ndx;
extern int crtimes_ndx;
@@ -1367,11 +1368,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
extra_len += EXTRA_LEN;
#endif
- if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
- file_checksum(thisname, &st, tmp_sum);
- if (sender_keeps_checksum)
- extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
- }
+ if (sender_keeps_checksum && S_ISREG(st.st_mode))
+ extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
#if EXTRA_ROUNDING > 0
if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
@@ -1460,8 +1458,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
return NULL;
}
- if (sender_keeps_checksum && S_ISREG(st.st_mode))
- memcpy(F_SUM(file), tmp_sum, flist_csum_len);
+ if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
+ if (!use_db || !db_get_checksum(&st, tmp_sum))
+ file_checksum(thisname, &st, tmp_sum);
+ if (sender_keeps_checksum)
+ memcpy(F_SUM(file), tmp_sum, flist_csum_len);
+ }
if (unsort_ndx)
F_NDX(file) = stats.num_dirs;
@@ -2145,6 +2147,9 @@ void send_extra_file_list(int f, int at_least)
finish:
if (io_error != save_io_error && protocol_version == 30 && !ignore_errors)
send_msg_int(MSG_IO_ERROR, io_error);
+
+ if (use_db && flist_eof)
+ db_disconnect(True);
}
struct file_list *send_file_list(int f, int argc, char *argv[])
@@ -2168,6 +2173,13 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
| (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
int implied_dot_dir = 0;
+ if (use_db) {
+ if (always_checksum)
+ db_connect(0); /* Will reset use_db on error. */
+ else
+ use_db = 0;
+ }
+
rprintf(FLOG, "building file list\n");
if (show_filelist_progress)
start_filelist_progress("building file list");
@@ -2511,6 +2523,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i());
}
+ if (use_db && (!inc_recurse || flist_eof))
+ db_disconnect(True);
+
return flist;
}
diff --git a/generator.c b/generator.c
--- a/generator.c
+++ b/generator.c
@@ -61,6 +61,7 @@ extern int ignore_non_existing;
extern int want_xattr_optim;
extern int modify_window;
extern int inplace;
+extern int use_db;
extern int append_mode;
extern int make_backups;
extern int csum_length;
@@ -610,7 +611,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
of the file time to determine whether to sync */
if (always_checksum > 0 && S_ISREG(st->st_mode)) {
char sum[MAX_DIGEST_LEN];
- file_checksum(fn, st, sum);
+ if (!use_db || !db_get_checksum(st, sum))
+ file_checksum(fn, st, sum);
return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
}
@@ -2268,6 +2270,13 @@ void generate_files(int f_out, const char *local_name)
: "enabled");
}
+ if (use_db) {
+ if (always_checksum || (append_mode != 1 && protocol_version >= 30))
+ db_connect(0); /* Will reset use_db on error. */
+ else
+ use_db = 0;
+ }
+
dflt_perms = (ACCESSPERMS & ~orig_umask);
do {
@@ -2393,6 +2402,9 @@ void generate_files(int f_out, const char *local_name)
wait_for_receiver();
}
+ if (use_db)
+ db_disconnect(True);
+
info_levels[INFO_FLIST] = save_info_flist;
info_levels[INFO_PROGRESS] = save_info_progress;
diff --git a/io.c b/io.c
--- a/io.c
+++ b/io.c
@@ -41,8 +41,10 @@ extern int am_server;
extern int am_sender;
extern int am_receiver;
extern int am_generator;
+extern int local_server;
extern int msgs2stderr;
extern int inc_recurse;
+extern int same_db;
extern int io_error;
extern int batch_fd;
extern int eol_nulls;
@@ -1492,6 +1494,32 @@ static void read_a_msg(void)
if (am_sender)
maybe_send_keepalive(time(NULL), MSK_ALLOW_FLUSH);
break;
+ case MSG_CHECKSUM:
+ /* This receives some checksum info that we want to make a note of
+ * (which allows a single process to do all the writing to the db). */
+ if (msg_bytes != MSG_CHECKSUM_LEN)
+ goto overflow;
+ raw_read_buf(data, MSG_CHECKSUM_LEN);
+ if (am_generator && same_db) {
+ iobuf.in_multiplexed = 1;
+ send_msg(MSG_CHECKSUM, data, MSG_CHECKSUM_LEN, 0);
+ } if (am_receiver || (am_sender && !local_server))
+ goto unexpected;
+ else {
+ /* The received data is a set of numbers followed by the checksum. */
+ STRUCT_STAT st;
+ st.st_dev = IVAL64(data, 0);
+ st.st_ino = IVAL64(data, 8);
+ st.st_size = IVAL64(data, 16);
+ st.st_mtime = IVAL64(data, 24);
+ st.st_ctime = IVAL64(data, 32);
+#if MSG_CHECKSUM_LONGS != 5
+#error Fix the parsing of checksum long values
+#endif
+ iobuf.in_multiplexed = 1;
+ db_set_checksum(IVAL(data, MSG_CHECKSUM_LONGS*8), &st, data + MSG_CHECKSUM_LONGS*8 + 4);
+ }
+ break;
case MSG_DELETED:
if (msg_bytes >= sizeof data)
goto overflow;
@@ -1643,6 +1671,7 @@ static void read_a_msg(void)
* with a duplicate exit message. */
_exit_cleanup(val, __FILE__, 0 - __LINE__);
default:
+ unexpected:
rprintf(FERROR, "unexpected tag %d [%s%s]\n",
tag, who_am_i(), inc_recurse ? "/inc" : "");
exit_cleanup(RERR_STREAMIO);
diff --git a/main.c b/main.c
--- a/main.c
+++ b/main.c
@@ -39,6 +39,7 @@ extern int am_root;
extern int am_server;
extern int am_sender;
extern int am_daemon;
+extern int am_dbadmin;
extern int inc_recurse;
extern int blocking_io;
extern int always_checksum;
@@ -57,6 +58,7 @@ extern int copy_unsafe_links;
extern int keep_dirlinks;
extern int preserve_hard_links;
extern int protocol_version;
+extern int always_checksum;
extern int mkpath_dest_arg;
extern int file_total;
extern int recurse;
@@ -93,6 +95,7 @@ extern char *logfile_format;
extern char *filesfrom_host;
extern char *partial_dir;
extern char *rsync_path;
+extern char *db_config;
extern char *shell_cmd;
extern char *password_file;
extern char *backup_dir;
@@ -1241,6 +1244,9 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
if (am_daemon && io_timeout && protocol_version >= 31)
send_msg_int(MSG_IO_TIMEOUT, io_timeout);
+ if (db_config)
+ db_read_config(FERROR, db_config);
+
if (am_sender) {
keep_dirlinks = 0; /* Must be disabled on the sender. */
if (need_messages_from_generator)
@@ -1531,6 +1537,9 @@ static int start_client(int argc, char *argv[])
else
env_port = rsync_port;
+ if (db_config)
+ db_read_config(FERROR, db_config);
+
if (daemon_connection < 0)
return start_socket_client(shell_machine, remote_argc, remote_argv, argc, argv);
diff --git a/options.c b/options.c
--- a/options.c
+++ b/options.c
@@ -83,6 +83,7 @@ int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
int am_server = 0;
int am_sender = 0;
int am_starting_up = 1;
+int am_dbadmin = 0;
int relative_paths = -1;
int implied_dirs = 1;
int missing_args = 0; /* 0 = FERROR_XFER, 1 = ignore, 2 = delete */
@@ -96,6 +97,7 @@ int use_qsort = 0;
char *files_from = NULL;
int filesfrom_fd = -1;
char *filesfrom_host = NULL;
+char *db_config = NULL;
int eol_nulls = 0;
int protect_args = -1;
int human_readable = 1;
@@ -104,6 +106,9 @@ int mkpath_dest_arg = 0;
int allow_inc_recurse = 1;
int xfer_dirs = -1;
int am_daemon = 0;
+int db_clean, db_check, db_do_md4, db_do_md5, db_update = 1, db_lax, db_init, db_mounts;
+int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
+int saw_db_output_opt, saw_db_sum_opt;
int connect_timeout = 0;
int keep_partial = 0;
int safe_symlinks = 0;
@@ -282,6 +287,7 @@ static struct output_struct debug_words[COUNT_DEBUG+1] = {
DEBUG_WORD(CHDIR, W_CLI|W_SRV, "Debug when the current directory changes"),
DEBUG_WORD(CONNECT, W_CLI, "Debug connection events (levels 1-2)"),
DEBUG_WORD(CMD, W_CLI, "Debug commands+options that are issued (levels 1-2)"),
+ DEBUG_WORD(DB, W_SND|W_REC, "Debug DB operations (levels 1-5)"),
DEBUG_WORD(DEL, W_REC, "Debug delete actions (levels 1-3)"),
DEBUG_WORD(DELTASUM, W_SND|W_REC, "Debug delta-transfer checksumming (levels 1-4)"),
DEBUG_WORD(DUP, W_REC, "Debug weeding of duplicate names"),
@@ -573,6 +579,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE,
+ OPT_NO_DB, OPT_DBONLY,
OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR,
OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS,
OPT_STOP_AFTER, OPT_STOP_AT,
@@ -729,6 +736,10 @@ static struct poptOption long_options[] = {
{"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
{"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
{"cc", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
+ {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
+ {"no-db", 0, POPT_ARG_NONE, 0, OPT_NO_DB, 0, 0 },
+ {"db-lax", 0, POPT_ARG_VAL, &db_lax, 1, 0, 0 },
+ {"no-db-lax", 0, POPT_ARG_VAL, &db_lax, 0, 0, 0 },
{"block-size", 'B', POPT_ARG_STRING, 0, OPT_BLOCK_SIZE, 0, 0 },
{"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
{"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
@@ -825,6 +836,9 @@ static struct poptOption long_options[] = {
{"dparam", 0, POPT_ARG_STRING, 0, OPT_DAEMON, 0, 0 },
{"detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 },
{"no-detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 },
+ /* All the following options switch us into DB-admin option-parsing. */
+ {"db-help", 0, POPT_ARG_NONE, 0, OPT_DBONLY, 0, 0 },
+ {"db-only", 0, POPT_ARG_STRING, 0, OPT_DBONLY, 0, 0 },
{0,0,0,0, 0, 0, 0}
};
@@ -853,6 +867,31 @@ static struct poptOption long_daemon_options[] = {
{0,0,0,0, 0, 0, 0}
};
+static struct poptOption long_dbonly_options[] = {
+ /* longName, shortName, argInfo, argPtr, value, descrip, argDesc */
+ {"check", 'c', POPT_ARG_NONE, &db_check, 0, 0, 0},
+ {"clean", 0, POPT_ARG_NONE, &db_clean, 0, 0, 0},
+ {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
+ {"db-only", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
+ {"db-lax", 0, POPT_ARG_VAL, &db_lax, 1, 0, 0 },
+ {"no-db-lax", 0, POPT_ARG_VAL, &db_lax, 0, 0, 0 },
+ {"info", 0, POPT_ARG_STRING, 0, OPT_INFO, 0, 0 },
+ {"debug", 0, POPT_ARG_STRING, 0, OPT_DEBUG, 0, 0 },
+ {"update", 'u', POPT_ARG_VAL, &db_update, 1, 0, 0 },
+ {"no-update", 'N', POPT_ARG_VAL, &db_update, 0, 0, 0 },
+ {"no-u", 0, POPT_ARG_VAL, &db_update, 0, 0, 0 },
+ {"output", 'o', POPT_ARG_STRING, 0, 'o', 0, 0 },
+ {"recursive", 'r', POPT_ARG_VAL, &recurse, 1, 0, 0 },
+ {"no-recursive", 0, POPT_ARG_VAL, &recurse, 0, 0, 0 },
+ {"no-r", 0, POPT_ARG_VAL, &recurse, 0, 0, 0 },
+ {"sums", 's', POPT_ARG_STRING, 0, 's', 0, 0 },
+ {"init", 0, POPT_ARG_NONE, &db_init, 0, 0, 0 },
+ {"mounts", 0, POPT_ARG_NONE, &db_mounts, 0, 0, 0 },
+ {"quiet", 'q', POPT_ARG_NONE, &quiet, 0, 0, 0 },
+ {"help", 'h', POPT_ARG_NONE, 0, 'h', 0, 0 },
+ {"db-help", 0, POPT_ARG_NONE, 0, 'h', 0, 0 },
+ {0,0,0,0, 0, 0, 0}
+};
static char err_buf[200];
@@ -978,6 +1017,8 @@ static void set_refuse_options(void)
parse_one_refuse_match(0, "iconv", list_end);
#endif
parse_one_refuse_match(0, "log-file*", list_end);
+ parse_one_refuse_match(0, "db", list_end);
+ parse_one_refuse_match(0, "db-lax", list_end);
}
#ifndef SUPPORT_ATIMES
@@ -1285,6 +1326,102 @@ static void create_refuse_error(int which)
snprintf(err_buf + n, sizeof err_buf - n, " (-%c)\n", op->shortName);
}
+static NORETURN void parse_dbonly_args(int argc, const char **argv)
+{
+ poptContext pc = poptGetContext(RSYNC_NAME, argc, argv, long_dbonly_options, 0);
+ const char *arg;
+ int opt;
+
+ recurse = 1;
+ am_dbadmin = 1;
+
+ while ((opt = poptGetNextOpt(pc)) != -1) {
+ const char *cp;
+ switch (opt) {
+ case 'o':
+ for (cp = poptGetOptArg(pc); *cp; cp++) {
+ switch (toLower(cp)) {
+ case 'n':
+ db_output_name = 1;
+ break;
+ case 's':
+ case 'c':
+ db_output_sum = db_output_name = 1;
+ break;
+ case 'i':
+ db_output_info = db_output_name = 1;
+ break;
+ case 'u':
+ db_output_unchanged = db_output_name = 1;
+ break;
+ case 'd':
+ db_output_dirs = 1;
+ break;
+ }
+ }
+ saw_db_output_opt = 1;
+ break;
+
+ case 's':
+ for (cp = poptGetOptArg(pc); *cp; cp++) {
+ switch (*cp) {
+ case '4':
+ db_do_md4 = 1;
+ break;
+ case '5':
+ db_do_md5 = 1;
+ break;
+ }
+ }
+ saw_db_sum_opt = 1;
+ break;
+
+ case 'h':
+ dbonly_usage(FINFO);
+ exit_cleanup(0);
+
+ case OPT_INFO:
+ arg = poptGetOptArg(pc);
+ parse_output_words(info_words, info_levels, arg, USER_PRIORITY);
+ break;
+
+ case OPT_DEBUG:
+ arg = poptGetOptArg(pc);
+ parse_output_words(debug_words, debug_levels, arg, USER_PRIORITY);
+ break;
+
+ default:
+ rprintf(FERROR,
+ "rsyncdb: %s: %s\n",
+ poptBadOption(pc, POPT_BADOPTION_NOALIAS),
+ poptStrerror(opt));
+ goto dbonly_usage;
+ }
+ }
+
+ if (!db_config) {
+ rprintf(FERROR, "You must specify the --db=FILE option.\n");
+ dbonly_usage:
+ rprintf(FERROR,
+ "(Type \"rsyncdb --help\" for assistance.)\n");
+ exit_cleanup(RERR_SYNTAX);
+ }
+
+ if (!saw_db_output_opt && !quiet) {
+ db_output_dirs = db_output_name = 1;
+ if (db_check)
+ db_output_info = 1;
+ }
+ if (!quiet)
+ db_output_msgs = 1;
+ if (!saw_db_sum_opt)
+ db_do_md5 = 1;
+
+ am_starting_up = 0;
+ run_dbonly(poptGetArgs(pc));
+ exit(42); /* NOT REACHED */
+}
+
/* This is used to make sure that --daemon & --server cannot be aliased to
* something else. These options have always disabled popt aliases for the
* parsing of a daemon or server command-line, but we have to make sure that
@@ -1341,6 +1478,12 @@ int parse_arguments(int *argc_p, const char ***argv_p)
return 0;
}
+ arg = *argv + strlen(*argv);
+ if (arg - *argv > 2 && strcmp(arg-2, "db") == 0) {
+ parse_dbonly_args(argc, argv);
+ /* NOT REACHED */
+ }
+
set_refuse_options();
#ifdef ICONV_OPTION
@@ -1459,6 +1602,12 @@ int parse_arguments(int *argc_p, const char ***argv_p)
am_daemon = 1;
return 1;
+ case OPT_DBONLY:
+ protect_args = 0;
+ poptFreeContext(pc);
+ parse_dbonly_args(argc, argv);
+ break; /* NOT REACHED */
+
case OPT_MODIFY_WINDOW:
/* The value has already been set by popt, but
* we need to remember that we're using a
@@ -1531,6 +1680,10 @@ int parse_arguments(int *argc_p, const char ***argv_p)
preserve_devices = preserve_specials = 0;
break;
+ case OPT_NO_DB:
+ db_config = NULL;
+ break;
+
case 'h':
human_readable++;
break;
diff --git a/pipe.c b/pipe.c
--- a/pipe.c
+++ b/pipe.c
@@ -27,11 +27,16 @@ extern int am_server;
extern int blocking_io;
extern int filesfrom_fd;
extern int munge_symlinks;
+extern int always_checksum;
+extern int use_db;
+extern char *db_config;
extern char *logfile_name;
extern int remote_option_cnt;
extern const char **remote_options;
extern struct chmod_mode_struct *chmod_modes;
+int same_db = 0;
+
/**
* Create a child connected to us via its stdin/stdout.
*
@@ -141,13 +146,22 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
}
if (remote_option_cnt) {
+ const char *db_config_save = db_config;
int rc = remote_option_cnt + 1;
const char **rv = remote_options;
if (!parse_arguments(&rc, &rv)) {
option_error();
exit_cleanup(RERR_SYNTAX);
}
- }
+ if (db_config == db_config_save)
+ same_db = db_config != NULL;
+ else if (!db_config || !db_config_save || strcmp(db_config, db_config_save) != 0) {
+ use_db = 0;
+ if (db_config)
+ db_read_config(FERROR, db_config);
+ }
+ } else if (use_db)
+ same_db = 1;
if (dup2(to_child_pipe[0], STDIN_FILENO) < 0
|| close(to_child_pipe[1]) < 0
diff --git a/receiver.c b/receiver.c
--- a/receiver.c
+++ b/receiver.c
@@ -24,6 +24,8 @@
extern int dry_run;
extern int do_xfers;
+extern int use_db;
+extern int db_lax;
extern int am_root;
extern int am_server;
extern int inc_recurse;
@@ -433,6 +435,11 @@ static void handle_delayed_updates(char *local_name)
"rename failed for %s (from %s)",
full_fname(fname), partialptr);
} else {
+ if (use_db && !db_lax) {
+ STRUCT_STAT st;
+ if (do_lstat(fname, &st) == 0)
+ db_update_ctime(5, &st);
+ }
if (remove_source_files
|| (preserve_hard_links && F_IS_HLINKED(file)))
send_msg_int(MSG_SUCCESS, ndx);
@@ -539,6 +546,9 @@ int recv_files(int f_in, int f_out, char *local_name)
if (delay_updates)
delayed_bits = bitbag_create(cur_flist->used + 1);
+ if (use_db && (append_mode == 1 || protocol_version < 30))
+ use_db = 0; /* We can't note finished md5 values */
+
progress_init();
while (1) {
@@ -878,6 +888,8 @@ int recv_files(int f_in, int f_out, char *local_name)
do_unlink(partialptr);
handle_partial_dir(partialptr, PDIR_DELETE);
}
+ if (use_db && do_lstat(fname, &st) == 0)
+ db_set_checksum(5, &st, sender_file_sum);
} else if (keep_partial && partialptr && !one_inplace) {
if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
rprintf(FERROR,
@@ -891,6 +903,8 @@ int recv_files(int f_in, int f_out, char *local_name)
recv_ok = -1;
else if (delay_updates && recv_ok) {
bitbag_set_bit(delayed_bits, ndx);
+ if (use_db && do_lstat(partialptr, &st) == 0)
+ db_set_checksum(5, &st, sender_file_sum);
recv_ok = 2;
} else
partialptr = NULL;
diff --git a/rsync.1.md b/rsync.1.md
--- a/rsync.1.md
+++ b/rsync.1.md
@@ -384,6 +384,9 @@ detailed description below for a complete description.
--dry-run, -n perform a trial run with no changes made
--whole-file, -W copy files whole (w/o delta-xfer algorithm)
--checksum-choice=STR choose the checksum algorithm (aka --cc)
+--db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
+--db-only=CONFIG_FILE behave like rsyncdb
+--db-lax ignore ctime changes (use with CAUTION)
--one-file-system, -x don't cross filesystem boundaries
--block-size=SIZE, -B force a fixed checksum block-size
--rsh=COMMAND, -e specify the remote shell to use
diff --git a/rsync.c b/rsync.c
--- a/rsync.c
+++ b/rsync.c
@@ -39,6 +39,7 @@ extern int am_daemon;
extern int am_sender;
extern int am_receiver;
extern int am_generator;
+extern int am_dbadmin;
extern int am_starting_up;
extern int allow_8bit_chars;
extern int protocol_version;
@@ -807,6 +808,8 @@ struct file_list *flist_for_ndx(int ndx, const char *fatal_error_loc)
const char *who_am_i(void)
{
+ if (am_dbadmin)
+ return "rsyncdb";
if (am_starting_up)
return am_server ? "server" : "client";
return am_sender ? "sender"
diff --git a/rsync.h b/rsync.h
--- a/rsync.h
+++ b/rsync.h
@@ -261,12 +261,16 @@ enum msgcode {
MSG_IO_ERROR=22,/* the sending side had an I/O error */
MSG_IO_TIMEOUT=33,/* tell client about a daemon's timeout value */
MSG_NOOP=42, /* a do-nothing message (legacy protocol-30 only) */
+ MSG_CHECKSUM=55,/* sent via rcvr -> gen pipe and local-host-only gen -> sender */
MSG_ERROR_EXIT=86, /* synchronize an error exit (siblings and protocol >= 31) */
MSG_SUCCESS=100,/* successfully updated indicated flist index */
MSG_DELETED=101,/* successfully deleted a file on receiving side */
MSG_NO_SEND=102,/* sender failed to open a file we wanted */
};
+#define MSG_CHECKSUM_LONGS 5
+#define MSG_CHECKSUM_LEN (MSG_CHECKSUM_LONGS*8 + 4 + MAX_DIGEST_LEN)
+
#define NDX_DONE -1
#define NDX_FLIST_EOF -2
#define NDX_DEL_STATS -3
@@ -1419,7 +1423,8 @@ extern short info_levels[], debug_levels[];
#define DEBUG_CHDIR (DEBUG_BIND+1)
#define DEBUG_CONNECT (DEBUG_CHDIR+1)
#define DEBUG_CMD (DEBUG_CONNECT+1)
-#define DEBUG_DEL (DEBUG_CMD+1)
+#define DEBUG_DB (DEBUG_CMD+1)
+#define DEBUG_DEL (DEBUG_DB+1)
#define DEBUG_DELTASUM (DEBUG_DEL+1)
#define DEBUG_DUP (DEBUG_DELTASUM+1)
#define DEBUG_EXIT (DEBUG_DUP+1)
diff --git a/rsyncdb-mountinfo b/rsyncdb-mountinfo
new file mode 100755
--- /dev/null
+++ b/rsyncdb-mountinfo
@@ -0,0 +1,82 @@
+#!/usr/bin/perl
+
+# This script outputs data for rsyncdb --mounts. It must output a complete
+# list of the mounts for the current host in a strict format -- 2 fields
+# with a Tab between: $MOUNT_UNIQ\t$PATH
+#
+# The list of mounts MUST NOT contain any entry that has the same devnum
+# (st_dev) as any other entry in the list (as checked via its PATH).
+#
+# MOUNT_UNIQ is a unique string that identifies the mount on this host.
+# This cannot be the devnum (st_dev) because that can vary depending on the
+# mount order or be reused for different mounts if they are not mounted at
+# the same time. Ideally this would be its UUID value, if that is available
+# on this OS. This script looks in /dev/disk/by-uuid for the current UUID
+# mappings). If the UUID is not found, the fallback default is the string
+# "Mount of $devname", which should be adequate for situations that don't
+# use removable media (though you may need to take steps to weed-out removable
+# mounts).
+#
+# You can override the MOUNT_UNIQ value by putting a .rsyncdb_mount_uniq
+# file in the root directory of any mount, at which point it is up to you
+# to make sure that the value stays unique (note that all sequences of
+# whitespace are transformed into a single space, and leading/trailing
+# whitespace is removed).
+#
+# MOUNT_UNIQ may never contain a Tab but it would be legal for PATH to have
+# a Tab (just really weird). Neither may have a CR or LF in it.
+#
+# The maximum size for MOUNT_UNIQ is 256 characters.
+#
+# If this script doesn't meet your needs, feel free to edit/replace it and
+# choose some other method of finding a unique value for each mount. If you
+# come up with a good idiom that might be useful to others, please share it
+# with the rsync mailing list.
+
+use strict;
+use warnings;
+use Cwd 'abs_path';
+
+my @MOUNT_FILES = qw( /proc/mounts /etc/mtab );
+my $VALID_DEVICE_REGEX = qr{^/dev|^rootfs$};
+my $UUID_DIR = '/dev/disk/by-uuid';
+my $OVERRIDE_FILE = '.rsyncdb_mount_uniq';
+
+my (%hash, %uuid);
+
+if (-d $UUID_DIR) {
+ foreach my $uuid (glob "$UUID_DIR/*") {
+ my $lnk = readlink($uuid);
+ if ($lnk !~ m{^/}) {
+ $lnk = abs_path("$UUID_DIR/$lnk");
+ }
+ $uuid =~ s{.*/}{};
+ $uuid{$lnk} = $uuid;
+ }
+}
+
+foreach my $mount_file (@MOUNT_FILES) {
+ if (open MOUNTS, $mount_file) {
+ while (<MOUNTS>) {
+ my ($devname, $path) = (split)[0,1];
+ next unless $devname =~ /$VALID_DEVICE_REGEX/;
+
+ my ($devno) = (stat($path))[0];
+ next unless defined $devno; # Skip if mount is invalid.
+ next if $hash{$devno}++; # SKip if we've seen this devno earlier.
+
+ my $mount_uniq = $uuid{$devname} ? $uuid{$devname} : "Mount of $devname";
+ if (open UNIQ, '<', "$path/$OVERRIDE_FILE") {
+ $mount_uniq = <UNIQ>;
+ close UNIQ;
+ $mount_uniq =~ s/\s+/ /g; # This ensures no tab, CR, nor LF.
+ $mount_uniq =~ s/^ | $//g; # .. and no leading or trailing whitespace.
+ }
+ print $mount_uniq, "\t", $path, "\n";
+ }
+ close MOUNTS;
+ exit;
+ }
+}
+
+die "Failed to to open any mount files: @MOUNT_FILES\n";
diff --git a/rsyncdb.1.md b/rsyncdb.1.md
new file mode 100644
--- /dev/null
+++ b/rsyncdb.1.md
@@ -0,0 +1,217 @@
+# NAME
+
+rsyncdb - Maintain an rsync checksum DB
+
+# SYNOPSIS
+
+```
+rsyncdb --db=CONFIG [OPTION...] [DIR...]
+```
+
+# DESCRIPTION
+
+Rsyncdb can maintain a checksum-caching DB that rsync can use to make its
+`--checksum` option more optimal. You must specify a config file via
+the `--db=CONFIG_FILE` option in order for rsyncdb to know what DB to
+manipulate. See the rsync manpage's `--db` option for full details on
+the file's format.
+
+You can specify one or more directory args for rsyncdb to scan. If no
+DIR args are specified, the current directory is assumed to be the spot
+to start scanning.
+
+Note that the rsyncdb program is usually just a symlink to the rsync program.
+You can force rsync to behave as rsyncdb either by having a symlink (or
+hardlink) name that ends with "db" or by `starting` the rsync args with
+`--db-only=CONFIG` (and that option works just like `--db=CONFIG` to
+a program named rsyncdb).
+
+# EXAMPLES
+
+The following command will update checksum information in the database
+described in the /etc/db.conf file:
+
+> rsyncdb --db=/etc/db.conf -o n --clean /dir1 /dir2
+
+It scans 2 directory hierarchies (/dir1 & /dir2) and cleans out any
+checksums whose inodes are no longer found in those directories (so that
+directory args are presumed to be complete for this host's DB contents).
+
+The following command will scan all the files in the /dir2 directory (without
+recursive scanning, due to the `--no-r` option) and check them against
+the DB:
+
+> rsyncdb --db=/etc/db.conf --check --no-r /dir2
+
+Any errors found are output as well as being fixed in the DB. (See
+`--no-update` for how to check without updating.)
+
+The following command will output MD5 sums for all the files found in the
+directories mentioned, even if they are unchanged (due to the
+`--output=us` option):
+
+> rsyncdb --db=/etc/db.conf -rous /dir* >/tmp/md5sums.txt
+
+This is just like running md5sum, only faster. Unlike md5sum, you can't
+specify a single file, so use `--no-r` and grep the output if you just
+want to see a single file's value.
+
+The following command initializes a new DB, and is required for any new DB:
+
+> rsyncdb --db=/etc/db.conf --init --mounts
+
+The `--init` option should only be used once (unless you want to
+destroy existing data). The `--mounts` option may need to be used
+periodically, and makes use of a helper script (see below).
+
+# OPTIONS SUMMARY
+
+Rsyncdb accepts the following options:
+
+[comment]: # (help-rsyncdb.h)
+
+```
+--db=CONFIG Specify the CONFIG file to read for the DB info
+--db-lax Ignore ctime changes (use with CAUTION)
+--recursive, -r Scan files in subdirs (the default w/o --no-recursive)
+--sums=SUMS, -s List which checksums to update (default: 4,5)
+--output=STR, -o One or more letters of what to output (default: "")
+--check, -c Check checksums (by reading the files) and fix any
+ issues. Makes --output default to "dni".
+--clean Note all inodes in the DIRS and remove DB extras
+--no-update, -N Avoids updating/adding info w/--check and/or --clean
+--init Initialize a DB by (re-)creating its tables
+--mounts Scan for mounted filesystems and update the DB
+--quiet, -q Disable the default non-error output
+--help, -h Display this help message
+```
+
+# OPTIONS
+
+Rsyncdb accepts both long (double-dash + word) and short (single-dash + letter)
+options. The full list of the available options are described below. If an
+option can be specified in more than one way, the choices are comma-separated.
+Some options only have a long variant, not a short. If the option takes a
+parameter, the parameter is only listed after the long variant, even though it
+must also be specified for the short. When specifying a parameter, you can
+either use the form --option=param or replace the '=' with whitespace. The
+parameter may need to be quoted in some manner for it to survive the shell's
+command-line parsing.
+
+0. `--db=CONFIG_FILE`
+
+ This tells rsyncdb what DB-config file to read for the DB setup. This is
+ the same as the option in rsync, so refer to that manpage for full details.
+
+0. `--db-lax`
+
+ This option works just like it does in rsync, so refer to that manpage for
+ full details.
+
+0. `--no-recursive, --no-r`
+
+ This disables the default recursive directory scan that is performed on the
+ listed directory args. The options `--recursive` and `-r` are also
+ accepted, if someone wants to override an earlier `--no-r` override.
+
+0. `--sums=SUMS, -s`
+
+ Only output/update the listed checksum types. By default we deal with just
+ the newer md5 checksums (i.e. `--sums=5`).
+
+ Note that this option does NOT affect the order that checksums are output
+ if "-o s" is enabled, so `-s5,4` is the same as `-s4,5`.
+
+0. `--output=STR, -o`
+
+ The output option lets you specify one or more letters indicating what
+ information should be output. If `--output` is not specified, the default
+ is either "dn" or (with `--check`) "dni".
+
+ The following letters are accepted in the string:
+
+ - `d` outputs "... dir_name ..." lines for each directory in our scan. if
+ "d" is omitted, then this progress indictor is not output.
+ - `n` includes the file's name in the per-file output. These lines are only
+ output for changed files unless "u" is given. The "n" option is implied
+ by every other output option letter except "d".
+ - `s` includes the checksum info in the per-file output.
+ - `c` is a synonym for 's'.
+ - `i` includes itemized change info in the per-file output.
+ - `!i` indicates that the time and/or size is wrong.
+ - `+4` indicates the MD4 sum is missing.
+ - `+5` indicates the MD5 sum is missing.
+ - `!4` indicates the MD4 sum is wrong.
+ - `!5` indicates the MD5 sum is wrong.
+ - `?4` indicates an unknown MD4 difference. This can happen if we didn't
+ need to read the file; i.e. if the time/size is wrong and no sum info
+ was requested.
+ - `?5` indicates an unknown MD5 difference.
+ - `u` includes unchanged files in the per-file output lines.
+
+0. `--check, -c`
+
+ Check the checksums (forcing the reading of all the files) and fix any
+ issues that are found. Makes `--output` default to "dni".
+
+0. `--clean`
+
+ Makes a temp-DB of all the inodes that we find in all the listed
+ directories and removes any extraneous checksums from the DB. You will
+ need to specify all the mounted directories that are present (and listed as
+ mounted) in the DB on this host or else the checksums from the unvisited
+ directories will be discarded from the DB. If you want to just --clean
+ without adding or updating the info of new or changed files, specify
+ `--no-update` as well.
+
+0. `--no-update, -N`
+
+ Avoids updating/adding info with `--check` and/or `--clean`.
+
+0. `--quiet, -q`
+
+ Disable the default (non-error) output settings. This turns off the
+ messages that `--init`, `--mount`, and `--clean` output, and makes the
+ default for `--output` be nothing (though an explicit `--output` option is
+ not affected).
+
+0. `--init`
+
+ Create the tables in the DB. If it is used on an existing DB, all the
+ existing tables are dropped and re-created.
+
+This option cannot be combined with the updating or reporting of checksum
+information, but may be combined with `--mounts`.
+
+0. `--mounts`
+
+ Populate the "disk" DB with the available device numbers and change any
+ mounted/unmount information for devices. This should be run every time a
+ mount-change happens that may affect a directory hierarchy in the DB.
+ Rsyncdb will not save any checksums for a device that is not listed in the
+ "disk" table.
+
+ The helper script "rsyncdb-mountinfo" is used as the source of the mount
+ information on the host, which it derives from various system files and
+ UUID directories (if available). That script supports the use of an
+ override file named ".rsyncdb_mount_uniq" in the root of the mount as one
+ way to manually assign unique values to a shared (mountable) device's
+ various disks.
+
+ Some advanced users may want to maintain the disk table themselves in order
+ to support mounting a drive in different (or multiple) locations, etc.
+
+ Specifying the `--mounts` option cannot be combined with updating or
+ reporting of checksum information, but may be combined with `--init`.
+
+0. `--help, -h`
+
+ Display a summary of the options.
+
+# SEE ALSO
+
+**rsync**(1)
+
+# AUTHOR
+
+Rsyncdb was written by Wayne Davison.
diff --git a/usage.c b/usage.c
--- a/usage.c
+++ b/usage.c
@@ -131,6 +131,16 @@ static void print_info_flags(enum logcode f)
#endif
"crtimes",
+#if !defined HAVE_MYSQL_MYSQL_H || !defined HAVE_LIBMYSQLCLIENT
+ "no "
+#endif
+ "MySQL",
+
+#if !defined HAVE_SQLITE3_H || !defined HAVE_LIBSQLITE3
+ "no "
+#endif
+ "SQLite",
+
"*Optimizations",
#ifndef HAVE_SIMD
@@ -250,6 +260,14 @@ void daemon_usage(enum logcode F)
rprintf(F,"daemon-specific rsync options. See also the rsyncd.conf(5) man page.\n");
}
+void dbonly_usage(enum logcode F)
+{
+ rprintf(F,"Usage: rsyncdb --db=CONFIG_FILE [OPTIONS] [DIRS]\n");
+ rprintf(F,"\n");
+ rprintf(F,"Options:\n");
+#include "help-rsyncdb.h"
+}
+
const char *rsync_version(void)
{
return RSYNC_GITVER;
diff -Nurp a/config.h.in b/config.h.in
--- a/config.h.in
+++ b/config.h.in
@@ -228,6 +228,9 @@
/* Define to 1 if you have the `inet' library (-linet). */
#undef HAVE_LIBINET
+/* Define to 1 if you have the `mysqlclient' library (-lmysqlclient). */
+#undef HAVE_LIBMYSQLCLIENT
+
/* Define to 1 if you have the `nsl' library (-lnsl). */
#undef HAVE_LIBNSL
@@ -246,6 +249,9 @@
/* Define to 1 if you have the `socket' library (-lsocket). */
#undef HAVE_LIBSOCKET
+/* Define to 1 if you have the `sqlite3' library (-lsqlite3). */
+#undef HAVE_LIBSQLITE3
+
/* Define to 1 if you have the `z' library (-lz). */
#undef HAVE_LIBZ
@@ -320,6 +326,9 @@
/* Define to 1 if you have the `mtrace' function. */
#undef HAVE_MTRACE
+/* Define to 1 if you have the <mysql/mysql.h> header file. */
+#undef HAVE_MYSQL_MYSQL_H
+
/* Define to 1 if you have the `nanosleep' function. */
#undef HAVE_NANOSLEEP
@@ -440,6 +449,15 @@
/* True if you have Solaris xattrs */
#undef HAVE_SOLARIS_XATTRS
+/* Define to 1 if you have the <sqlite3.h> header file. */
+#undef HAVE_SQLITE3_H
+
+/* Define to 1 if you have the `sqlite3_open_v2' function. */
+#undef HAVE_SQLITE3_OPEN_V2
+
+/* Define to 1 if you have the `sqlite3_prepare_v2' function. */
+#undef HAVE_SQLITE3_PREPARE_V2
+
/* Define to 1 if you have the <stdint.h> header file. */
#undef HAVE_STDINT_H
diff -Nurp a/configure.sh b/configure.sh
--- a/configure.sh
+++ b/configure.sh
@@ -625,6 +625,7 @@ ac_includes_default="\
ac_header_list=
ac_subst_vars='LTLIBOBJS
+MYSQL_CONFIG
MAKE_MAN
BUILD_ZLIB
BUILD_POPT
@@ -733,6 +734,8 @@ enable_iconv_open
enable_iconv
enable_acl_support
enable_xattr_support
+enable_mysql
+enable_sqlite
'
ac_precious_vars='build_alias
host_alias
@@ -1388,6 +1391,8 @@ Optional Features:
--disable-iconv disable rsync's --iconv option
--disable-acl-support disable ACL support
--disable-xattr-support disable extended attributes
+ --disable-mysql disable mysql DB support
+ --disable-sqlite disable sqlite DB support
Optional Packages:
--with-PACKAGE[=ARG] use PACKAGE [ARG=yes]
@@ -6249,6 +6254,7 @@ for ac_header in sys/fcntl.h sys/select.
unistd.h utime.h grp.h compat.h sys/param.h ctype.h sys/wait.h \
sys/ioctl.h sys/filio.h string.h stdlib.h sys/socket.h sys/mode.h \
sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
+ mysql/mysql.h sqlite3.h \
netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h dl.h \
popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netinet/ip.h \
@@ -10086,6 +10092,196 @@ $as_echo "$rsync_warn_flag" >&6; }
fi
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to include mysql DB support" >&5
+$as_echo_n "checking whether to include mysql DB support... " >&6; }
+# Check whether --enable-mysql was given.
+if test "${enable_mysql+set}" = set; then :
+ enableval=$enable_mysql;
+fi
+
+
+if test x"$enable_mysql" = x"no"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ # Extract the first word of "mysql_config", so it can be a program name with args.
+set dummy mysql_config; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_MYSQL_CONFIG+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$MYSQL_CONFIG"; then
+ ac_cv_prog_MYSQL_CONFIG="$MYSQL_CONFIG" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_prog_MYSQL_CONFIG="1"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ test -z "$ac_cv_prog_MYSQL_CONFIG" && ac_cv_prog_MYSQL_CONFIG="0"
+fi
+fi
+MYSQL_CONFIG=$ac_cv_prog_MYSQL_CONFIG
+if test -n "$MYSQL_CONFIG"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MYSQL_CONFIG" >&5
+$as_echo "$MYSQL_CONFIG" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ if test x$MYSQL_CONFIG = x1; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mysql version >= 4" >&5
+$as_echo_n "checking for mysql version >= 4... " >&6; }
+ mysql_version=`mysql_config --version`
+ mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
+ if test $mysql_major_version -lt 4; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no.. skipping MySQL" >&5
+$as_echo "no.. skipping MySQL" >&6; }
+ else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+ MYSQL_CFLAGS=`mysql_config --cflags`
+ MYSQL_LIBS=`mysql_config --libs`
+
+ CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
+ LIBS="$MYSQL_LIBS $LIBS"
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mysql_init in -lmysqlclient" >&5
+$as_echo_n "checking for mysql_init in -lmysqlclient... " >&6; }
+if ${ac_cv_lib_mysqlclient_mysql_init+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmysqlclient $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mysql_init ();
+int
+main ()
+{
+return mysql_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mysqlclient_mysql_init=yes
+else
+ ac_cv_lib_mysqlclient_mysql_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mysqlclient_mysql_init" >&5
+$as_echo "$ac_cv_lib_mysqlclient_mysql_init" >&6; }
+if test "x$ac_cv_lib_mysqlclient_mysql_init" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBMYSQLCLIENT 1
+_ACEOF
+
+ LIBS="-lmysqlclient $LIBS"
+
+fi
+
+ fi
+ fi
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to include sqlite DB support" >&5
+$as_echo_n "checking whether to include sqlite DB support... " >&6; }
+# Check whether --enable-sqlite was given.
+if test "${enable_sqlite+set}" = set; then :
+ enableval=$enable_sqlite;
+fi
+
+
+if test x"$enable_sqlite" = x"no"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for sqlite3_open in -lsqlite3" >&5
+$as_echo_n "checking for sqlite3_open in -lsqlite3... " >&6; }
+if ${ac_cv_lib_sqlite3_sqlite3_open+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lsqlite3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char sqlite3_open ();
+int
+main ()
+{
+return sqlite3_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_sqlite3_sqlite3_open=yes
+else
+ ac_cv_lib_sqlite3_sqlite3_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_open" >&5
+$as_echo "$ac_cv_lib_sqlite3_sqlite3_open" >&6; }
+if test "x$ac_cv_lib_sqlite3_sqlite3_open" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBSQLITE3 1
+_ACEOF
+
+ LIBS="-lsqlite3 $LIBS"
+
+fi
+
+ for ac_func in sqlite3_open_v2 sqlite3_prepare_v2
+do :
+ as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
+ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
+if eval test \"x\$"$as_ac_var"\" = x"yes"; then :
+ cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+fi
+
case "$CC" in
' checker'*|checker*)
diff -Nurp a/rsync.1 b/rsync.1
--- a/rsync.1
+++ b/rsync.1
@@ -460,6 +460,9 @@ detailed description below for a complet
--dry-run, -n perform a trial run with no changes made
--whole-file, -W copy files whole (w/o delta-xfer algorithm)
--checksum-choice=STR choose the checksum algorithm (aka --cc)
+--db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
+--db-only=CONFIG_FILE behave like rsyncdb
+--db-lax ignore ctime changes (use with CAUTION)
--one-file-system, -x don't cross filesystem boundaries
--block-size=SIZE, -B force a fixed checksum block-size
--rsh=COMMAND, -e specify the remote shell to use
diff -Nurp a/rsync.1.html b/rsync.1.html
--- a/rsync.1.html
+++ b/rsync.1.html
@@ -375,6 +375,9 @@ detailed description below for a complet
--dry-run, -n perform a trial run with no changes made
--whole-file, -W copy files whole (w/o delta-xfer algorithm)
--checksum-choice=STR choose the checksum algorithm (aka --cc)
+--db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
+--db-only=CONFIG_FILE behave like rsyncdb
+--db-lax ignore ctime changes (use with CAUTION)
--one-file-system, -x don't cross filesystem boundaries
--block-size=SIZE, -B force a fixed checksum block-size
--rsh=COMMAND, -e specify the remote shell to use
diff -Nurp a/rsyncdb.1 b/rsyncdb.1
--- a/rsyncdb.1
+++ b/rsyncdb.1
@@ -0,0 +1,230 @@
+.TH "rsyncdb" "1" "06 Aug 2020" "rsyncdb 3.2.3" "User Commands"
+.P
+.SH "NAME"
+.P
+rsyncdb \- Maintain an rsync checksum DB
+.P
+.SH "SYNOPSIS"
+.P
+.nf
+rsyncdb --db=CONFIG [OPTION...] [DIR...]
+.fi
+.P
+.SH "DESCRIPTION"
+.P
+Rsyncdb can maintain a checksum-caching DB that rsync can use to make its
+\fB\-\-checksum\fP option more optimal. You must specify a config file via
+the \fB\-\-db=CONFIG_FILE\fP option in order for rsyncdb to know what DB to
+manipulate. See the rsync manpage's \fB\-\-db\fP option for full details on
+the file's format.
+.P
+You can specify one or more directory args for rsyncdb to scan. If no
+DIR args are specified, the current directory is assumed to be the spot
+to start scanning.
+.P
+Note that the rsyncdb program is usually just a symlink to the rsync program.
+You can force rsync to behave as rsyncdb either by having a symlink (or
+hardlink) name that ends with "db" or by \fBstarting\fP the rsync args with
+\fB\-\-db-only=CONFIG\fP (and that option works just like \fB\-\-db=CONFIG\fP to
+a program named rsyncdb).
+.P
+.SH "EXAMPLES"
+.P
+The following command will update checksum information in the database
+described in the /etc/db.conf file:
+.RS 4
+.P
+.nf
+rsyncdb --db=/etc/db.conf -o n --clean /dir1 /dir2
+.fi
+.RE
+.P
+It scans 2 directory hierarchies (/dir1 & /dir2) and cleans out any
+checksums whose inodes are no longer found in those directories (so that
+directory args are presumed to be complete for this host's DB contents).
+.P
+The following command will scan all the files in the /dir2 directory (without
+recursive scanning, due to the \fB\-\-no-r\fP option) and check them against
+the DB:
+.RS 4
+.P
+.nf
+rsyncdb --db=/etc/db.conf --check --no-r /dir2
+.fi
+.RE
+.P
+Any errors found are output as well as being fixed in the DB. (See
+\fB\-\-no-update\fP for how to check without updating.)
+.P
+The following command will output MD5 sums for all the files found in the
+directories mentioned, even if they are unchanged (due to the
+\fB\-\-output=us\fP option):
+.RS 4
+.P
+.nf
+rsyncdb --db=/etc/db.conf -rous /dir* >/tmp/md5sums.txt
+.fi
+.RE
+.P
+This is just like running md5sum, only faster. Unlike md5sum, you can't
+specify a single file, so use \fB\-\-no-r\fP and grep the output if you just
+want to see a single file's value.
+.P
+The following command initializes a new DB, and is required for any new DB:
+.RS 4
+.P
+.nf
+rsyncdb --db=/etc/db.conf --init --mounts
+.fi
+.RE
+.P
+The \fB\-\-init\fP option should only be used once (unless you want to
+destroy existing data). The \fB\-\-mounts\fP option may need to be used
+periodically, and makes use of a helper script (see below).
+.P
+.SH "OPTIONS SUMMARY"
+.P
+Rsyncdb accepts the following options:
+.P
+.nf
+--db=CONFIG Specify the CONFIG file to read for the DB info
+--db-lax Ignore ctime changes (use with CAUTION)
+--recursive, -r Scan files in subdirs (the default w/o --no-recursive)
+--sums=SUMS, -s List which checksums to update (default: 4,5)
+--output=STR, -o One or more letters of what to output (default: "")
+--check, -c Check checksums (by reading the files) and fix any
+ issues. Makes --output default to "dni".
+--clean Note all inodes in the DIRS and remove DB extras
+--no-update, -N Avoids updating/adding info w/--check and/or --clean
+--init Initialize a DB by (re-)creating its tables
+--mounts Scan for mounted filesystems and update the DB
+--quiet, -q Disable the default non-error output
+--help, -h Display this help message
+.fi
+.P
+.SH "OPTIONS"
+.P
+Rsyncdb accepts both long (double-dash + word) and short (single-dash + letter)
+options. The full list of the available options are described below. If an
+option can be specified in more than one way, the choices are comma-separated.
+Some options only have a long variant, not a short. If the option takes a
+parameter, the parameter is only listed after the long variant, even though it
+must also be specified for the short. When specifying a parameter, you can
+either use the form \-\-option=param or replace the '=' with whitespace. The
+parameter may need to be quoted in some manner for it to survive the shell's
+command-line parsing.
+.P
+.IP "\fB\-\-db=CONFIG_FILE\fP"
+This tells rsyncdb what DB-config file to read for the DB setup. This is
+the same as the option in rsync, so refer to that manpage for full details.
+.IP "\fB\-\-db-lax\fP"
+This option works just like it does in rsync, so refer to that manpage for
+full details.
+.IP "\fB\-\-no-recursive,\ \-\-no-r\fP"
+This disables the default recursive directory scan that is performed on the
+listed directory args. The options \fB\-\-recursive\fP and \fB\-r\fP are also
+accepted, if someone wants to override an earlier \fB\-\-no-r\fP override.
+.IP "\fB\-\-sums=SUMS,\ \-s\fP"
+Only output/update the listed checksum types. By default we deal with just
+the newer md5 checksums (i.e. \fB\-\-sums=5\fP).
+.IP
+Note that this option does NOT affect the order that checksums are output
+if "\-o s" is enabled, so \fB\-s5,4\fP is the same as \fB\-s4,5\fP.
+.IP "\fB\-\-output=STR,\ \-o\fP"
+The output option lets you specify one or more letters indicating what
+information should be output. If \fB\-\-output\fP is not specified, the default
+is either "dn" or (with \fB\-\-check\fP) "dni".
+.IP
+The following letters are accepted in the string:
+.IP
+.RS
+.IP o
+\fBd\fP outputs "... dir_name ..." lines for each directory in our scan. if
+"d" is omitted, then this progress indictor is not output.
+.IP o
+\fBn\fP includes the file's name in the per-file output. These lines are only
+output for changed files unless "u" is given. The "n" option is implied
+by every other output option letter except "d".
+.IP o
+\fBs\fP includes the checksum info in the per-file output.
+.IP o
+\fBc\fP is a synonym for 's'.
+.IP o
+.IP
+.RS
+.IP o
+\fBi\fP includes itemized change info in the per-file output.
+
+\fB!i\fP indicates that the time and/or size is wrong.
+.IP o
+\fB+4\fP indicates the MD4 sum is missing.
+.IP o
+\fB+5\fP indicates the MD5 sum is missing.
+.IP o
+\fB!4\fP indicates the MD4 sum is wrong.
+.IP o
+\fB!5\fP indicates the MD5 sum is wrong.
+.IP o
+\fB?4\fP indicates an unknown MD4 difference. This can happen if we didn't
+need to read the file; i.e. if the time/size is wrong and no sum info
+was requested.
+.IP o
+\fB?5\fP indicates an unknown MD5 difference.
+.RE
+.IP o
+\fBu\fP includes unchanged files in the per-file output lines.
+.RE
+.IP "\fB\-\-check,\ \-c\fP"
+Check the checksums (forcing the reading of all the files) and fix any
+issues that are found. Makes \fB\-\-output\fP default to "dni".
+.IP "\fB\-\-clean\fP"
+Makes a temp-DB of all the inodes that we find in all the listed
+directories and removes any extraneous checksums from the DB. You will
+need to specify all the mounted directories that are present (and listed as
+mounted) in the DB on this host or else the checksums from the unvisited
+directories will be discarded from the DB. If you want to just \-\-clean
+without adding or updating the info of new or changed files, specify
+\fB\-\-no-update\fP as well.
+.IP "\fB\-\-no-update,\ \-N\fP"
+Avoids updating/adding info with \fB\-\-check\fP and/or \fB\-\-clean\fP.
+.IP "\fB\-\-quiet,\ \-q\fP"
+Disable the default (non-error) output settings. This turns off the
+messages that \fB\-\-init\fP, \fB\-\-mount\fP, and \fB\-\-clean\fP output, and makes the
+default for \fB\-\-output\fP be nothing (though an explicit \fB\-\-output\fP option is
+not affected).
+.IP "\fB\-\-init\fP"
+Create the tables in the DB. If it is used on an existing DB, all the
+existing tables are dropped and re-created.
+.P
+This option cannot be combined with the updating or reporting of checksum
+information, but may be combined with \fB\-\-mounts\fP.
+.P
+.IP "\fB\-\-mounts\fP"
+Populate the "disk" DB with the available device numbers and change any
+mounted/unmount information for devices. This should be run every time a
+mount-change happens that may affect a directory hierarchy in the DB.
+Rsyncdb will not save any checksums for a device that is not listed in the
+"disk" table.
+.IP
+The helper script "rsyncdb-mountinfo" is used as the source of the mount
+information on the host, which it derives from various system files and
+UUID directories (if available). That script supports the use of an
+override file named ".rsyncdb_mount_uniq" in the root of the mount as one
+way to manually assign unique values to a shared (mountable) device's
+various disks.
+.IP
+Some advanced users may want to maintain the disk table themselves in order
+to support mounting a drive in different (or multiple) locations, etc.
+.IP
+Specifying the \fB\-\-mounts\fP option cannot be combined with updating or
+reporting of checksum information, but may be combined with \fB\-\-init\fP.
+.IP "\fB\-\-help,\ \-h\fP"
+Display a summary of the options.
+.P
+.SH "SEE ALSO"
+.P
+\fBrsync\fP(1)
+.P
+.SH "AUTHOR"
+.P
+Rsyncdb was written by Wayne Davison.
diff -Nurp a/rsyncdb.1.html b/rsyncdb.1.html
--- a/rsyncdb.1.html
+++ b/rsyncdb.1.html
@@ -0,0 +1,228 @@
+<html><head>
+<title>rsyncdb(1) man page</title>
+<link href="https://fonts.googleapis.com/css2?family=Roboto&family=Roboto+Mono&display=swap" rel="stylesheet">
+<style>
+body {
+ max-width: 50em;
+ margin: auto;
+}
+body, b, strong, u {
+ font-family: 'Roboto', sans-serif;
+}
+code {
+ font-family: 'Roboto Mono', monospace;
+ font-weight: bold;
+ white-space: pre;
+}
+pre code {
+ display: block;
+ font-weight: normal;
+}
+blockquote pre code {
+ background: #f1f1f1;
+}
+dd p:first-of-type {
+ margin-block-start: 0em;
+}
+</style>
+</head><body>
+<h1>NAME</h1>
+<p>rsyncdb -⁠ Maintain an rsync checksum DB</p>
+<h1>SYNOPSIS</h1>
+<pre><code>rsyncdb --db=CONFIG [OPTION...] [DIR...]
+</code></pre>
+<h1>DESCRIPTION</h1>
+<p>Rsyncdb can maintain a checksum-caching DB that rsync can use to make its
+<code>--checksum</code> option more optimal. You must specify a config file via
+the <code>--db=CONFIG_FILE</code> option in order for rsyncdb to know what DB to
+manipulate. See the rsync manpage's <code>--db</code> option for full details on
+the file's format.</p>
+<p>You can specify one or more directory args for rsyncdb to scan. If no
+DIR args are specified, the current directory is assumed to be the spot
+to start scanning.</p>
+<p>Note that the rsyncdb program is usually just a symlink to the rsync program.
+You can force rsync to behave as rsyncdb either by having a symlink (or
+hardlink) name that ends with "db" or by <code>starting</code> the rsync args with
+<code>--db-only=CONFIG</code> (and that option works just like <code>--db=CONFIG</code> to
+a program named rsyncdb).</p>
+<h1>EXAMPLES</h1>
+<p>The following command will update checksum information in the database
+described in the /etc/db.conf file:</p>
+<blockquote>
+<pre><code>rsyncdb --db=/etc/db.conf -o n --clean /dir1 /dir2
+</code></pre>
+</blockquote>
+<p>It scans 2 directory hierarchies (/dir1 & /dir2) and cleans out any
+checksums whose inodes are no longer found in those directories (so that
+directory args are presumed to be complete for this host's DB contents).</p>
+<p>The following command will scan all the files in the /dir2 directory (without
+recursive scanning, due to the <code>--no-r</code> option) and check them against
+the DB:</p>
+<blockquote>
+<pre><code>rsyncdb --db=/etc/db.conf --check --no-r /dir2
+</code></pre>
+</blockquote>
+<p>Any errors found are output as well as being fixed in the DB. (See
+<code>--no-update</code> for how to check without updating.)</p>
+<p>The following command will output MD5 sums for all the files found in the
+directories mentioned, even if they are unchanged (due to the
+<code>--output=us</code> option):</p>
+<blockquote>
+<pre><code>rsyncdb --db=/etc/db.conf -rous /dir* >/tmp/md5sums.txt
+</code></pre>
+</blockquote>
+<p>This is just like running md5sum, only faster. Unlike md5sum, you can't
+specify a single file, so use <code>--no-r</code> and grep the output if you just
+want to see a single file's value.</p>
+<p>The following command initializes a new DB, and is required for any new DB:</p>
+<blockquote>
+<pre><code>rsyncdb --db=/etc/db.conf --init --mounts
+</code></pre>
+</blockquote>
+<p>The <code>--init</code> option should only be used once (unless you want to
+destroy existing data). The <code>--mounts</code> option may need to be used
+periodically, and makes use of a helper script (see below).</p>
+<h1>OPTIONS SUMMARY</h1>
+<p>Rsyncdb accepts the following options:</p>
+<pre><code>--db=CONFIG Specify the CONFIG file to read for the DB info
+--db-lax Ignore ctime changes (use with CAUTION)
+--recursive, -r Scan files in subdirs (the default w/o --no-recursive)
+--sums=SUMS, -s List which checksums to update (default: 4,5)
+--output=STR, -o One or more letters of what to output (default: "")
+--check, -c Check checksums (by reading the files) and fix any
+ issues. Makes --output default to "dni".
+--clean Note all inodes in the DIRS and remove DB extras
+--no-update, -N Avoids updating/adding info w/--check and/or --clean
+--init Initialize a DB by (re-)creating its tables
+--mounts Scan for mounted filesystems and update the DB
+--quiet, -q Disable the default non-error output
+--help, -h Display this help message
+</code></pre>
+<h1>OPTIONS</h1>
+<p>Rsyncdb accepts both long (double-dash + word) and short (single-dash + letter)
+options. The full list of the available options are described below. If an
+option can be specified in more than one way, the choices are comma-separated.
+Some options only have a long variant, not a short. If the option takes a
+parameter, the parameter is only listed after the long variant, even though it
+must also be specified for the short. When specifying a parameter, you can
+either use the form -⁠-⁠option=param or replace the '=' with whitespace. The
+parameter may need to be quoted in some manner for it to survive the shell's
+command-line parsing.</p>
+<dl>
+
+<dt><code>--db=CONFIG_FILE</code></dt><dd>
+<p>This tells rsyncdb what DB-config file to read for the DB setup. This is
+the same as the option in rsync, so refer to that manpage for full details.</p>
+</dd>
+
+<dt><code>--db-lax</code></dt><dd>
+<p>This option works just like it does in rsync, so refer to that manpage for
+full details.</p>
+</dd>
+
+<dt><code>--no-recursive, --no-r</code></dt><dd>
+<p>This disables the default recursive directory scan that is performed on the
+listed directory args. The options <code>--recursive</code> and <code>-r</code> are also
+accepted, if someone wants to override an earlier <code>--no-r</code> override.</p>
+</dd>
+
+<dt><code>--sums=SUMS, -s</code></dt><dd>
+<p>Only output/update the listed checksum types. By default we deal with just
+the newer md5 checksums (i.e. <code>--sums=5</code>).</p>
+<p>Note that this option does NOT affect the order that checksums are output
+if "-⁠o s" is enabled, so <code>-s5,4</code> is the same as <code>-s4,5</code>.</p>
+</dd>
+
+<dt><code>--output=STR, -o</code></dt><dd>
+<p>The output option lets you specify one or more letters indicating what
+information should be output. If <code>--output</code> is not specified, the default
+is either "dn" or (with <code>--check</code>) "dni".</p>
+<p>The following letters are accepted in the string:</p>
+<ul>
+<li><code>d</code> outputs "... dir_name ..." lines for each directory in our scan. if
+"d" is omitted, then this progress indictor is not output.</li>
+<li><code>n</code> includes the file's name in the per-file output. These lines are only
+output for changed files unless "u" is given. The "n" option is implied
+by every other output option letter except "d".</li>
+<li><code>s</code> includes the checksum info in the per-file output.</li>
+<li><code>c</code> is a synonym for 's'.</li>
+<li><code>i</code> includes itemized change info in the per-file output.
+<ul>
+<li><code>!i</code> indicates that the time and/or size is wrong.</li>
+<li><code>+4</code> indicates the MD4 sum is missing.</li>
+<li><code>+5</code> indicates the MD5 sum is missing.</li>
+<li><code>!4</code> indicates the MD4 sum is wrong.</li>
+<li><code>!5</code> indicates the MD5 sum is wrong.</li>
+<li><code>?4</code> indicates an unknown MD4 difference. This can happen if we didn't
+need to read the file; i.e. if the time/size is wrong and no sum info
+was requested.</li>
+<li><code>?5</code> indicates an unknown MD5 difference.</li>
+</ul>
+</li>
+<li><code>u</code> includes unchanged files in the per-file output lines.</li>
+</ul>
+</dd>
+
+<dt><code>--check, -c</code></dt><dd>
+<p>Check the checksums (forcing the reading of all the files) and fix any
+issues that are found. Makes <code>--output</code> default to "dni".</p>
+</dd>
+
+<dt><code>--clean</code></dt><dd>
+<p>Makes a temp-DB of all the inodes that we find in all the listed
+directories and removes any extraneous checksums from the DB. You will
+need to specify all the mounted directories that are present (and listed as
+mounted) in the DB on this host or else the checksums from the unvisited
+directories will be discarded from the DB. If you want to just -⁠-⁠clean
+without adding or updating the info of new or changed files, specify
+<code>--no-update</code> as well.</p>
+</dd>
+
+<dt><code>--no-update, -N</code></dt><dd>
+<p>Avoids updating/adding info with <code>--check</code> and/or <code>--clean</code>.</p>
+</dd>
+
+<dt><code>--quiet, -q</code></dt><dd>
+<p>Disable the default (non-error) output settings. This turns off the
+messages that <code>--init</code>, <code>--mount</code>, and <code>--clean</code> output, and makes the
+default for <code>--output</code> be nothing (though an explicit <code>--output</code> option is
+not affected).</p>
+</dd>
+
+<dt><code>--init</code></dt><dd>
+<p>Create the tables in the DB. If it is used on an existing DB, all the
+existing tables are dropped and re-created.</p>
+</dd>
+</dl>
+<p>This option cannot be combined with the updating or reporting of checksum
+information, but may be combined with <code>--mounts</code>.</p>
+<dl>
+
+<dt><code>--mounts</code></dt><dd>
+<p>Populate the "disk" DB with the available device numbers and change any
+mounted/unmount information for devices. This should be run every time a
+mount-change happens that may affect a directory hierarchy in the DB.
+Rsyncdb will not save any checksums for a device that is not listed in the
+"disk" table.</p>
+<p>The helper script "rsyncdb-mountinfo" is used as the source of the mount
+information on the host, which it derives from various system files and
+UUID directories (if available). That script supports the use of an
+override file named ".rsyncdb_mount_uniq" in the root of the mount as one
+way to manually assign unique values to a shared (mountable) device's
+various disks.</p>
+<p>Some advanced users may want to maintain the disk table themselves in order
+to support mounting a drive in different (or multiple) locations, etc.</p>
+<p>Specifying the <code>--mounts</code> option cannot be combined with updating or
+reporting of checksum information, but may be combined with <code>--init</code>.</p>
+</dd>
+
+<dt><code>--help, -h</code></dt><dd>
+<p>Display a summary of the options.</p>
+</dd>
+</dl>
+<h1>SEE ALSO</h1>
+<p><strong>rsync</strong>(1)</p>
+<h1>AUTHOR</h1>
+<p>Rsyncdb was written by Wayne Davison.</p>
+<div style="float: right"><p><i>06 Aug 2020</i></p></div>
+</body></html>
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to db.diff CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / rsync / patches