Annotation of embedaddon/rsync/patches/checksum-reading.diff, revision 1.1
1.1 ! misho 1: Optimize the --checksum option using externally created .rsyncsums files.
! 2:
! 3: This adds a new option, --sumfiles=MODE, that allows you to use a cache of
! 4: checksums when performing a --checksum transfer. These checksum files
! 5: (.rsyncsums) must be created by some other process -- see the perl script,
! 6: rsyncsums, in the support dir for one way.
! 7:
! 8: This option can be particularly helpful to a public mirror that wants to
! 9: pre-compute their .rsyncsums files, set the "checksum files = strict" option
! 10: in their daemon config file, and thus make it quite efficient for a client
! 11: rsync to make use of the --checksum option on their server.
! 12:
! 13: To use this patch, run these commands for a successful build:
! 14:
! 15: patch -p1 <patches/checksum-reading.diff
! 16: ./configure (optional if already run)
! 17: make
! 18:
! 19: based-on: e94bad1c156fc3910f24e2b3b71a81b0b0bdeb70
! 20: diff --git a/clientserver.c b/clientserver.c
! 21: --- a/clientserver.c
! 22: +++ b/clientserver.c
! 23: @@ -44,6 +44,8 @@ extern int numeric_ids;
! 24: extern int filesfrom_fd;
! 25: extern int remote_protocol;
! 26: extern int protocol_version;
! 27: +extern int always_checksum;
! 28: +extern int checksum_files;
! 29: extern int io_timeout;
! 30: extern int no_detach;
! 31: extern int write_batch;
! 32: @@ -1033,6 +1035,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
! 33: } else if (am_root < 0) /* Treat --fake-super from client as --super. */
! 34: am_root = 2;
! 35:
! 36: + checksum_files = always_checksum ? lp_checksum_files(i)
! 37: + : CSF_IGNORE_FILES;
! 38: +
! 39: if (filesfrom_fd == 0)
! 40: filesfrom_fd = f_in;
! 41:
! 42: diff --git a/daemon-parm.txt b/daemon-parm.txt
! 43: --- a/daemon-parm.txt
! 44: +++ b/daemon-parm.txt
! 45: @@ -49,6 +49,7 @@ INTEGER max_connections 0
! 46: INTEGER max_verbosity 1
! 47: INTEGER timeout 0
! 48:
! 49: +ENUM checksum_files CSF_IGNORE_FILES
! 50: ENUM syslog_facility LOG_DAEMON
! 51:
! 52: BOOL fake_super False
! 53: diff --git a/flist.c b/flist.c
! 54: --- a/flist.c
! 55: +++ b/flist.c
! 56: @@ -22,6 +22,7 @@
! 57:
! 58: #include "rsync.h"
! 59: #include "ifuncs.h"
! 60: +#include "itypes.h"
! 61: #include "rounding.h"
! 62: #include "inums.h"
! 63: #include "io.h"
! 64: @@ -33,6 +34,7 @@ extern int am_sender;
! 65: extern int am_generator;
! 66: extern int inc_recurse;
! 67: extern int always_checksum;
! 68: +extern int basis_dir_cnt;
! 69: extern int checksum_type;
! 70: extern int module_id;
! 71: extern int ignore_errors;
! 72: @@ -62,6 +64,7 @@ extern int implied_dirs;
! 73: extern int ignore_perishable;
! 74: extern int non_perishable_cnt;
! 75: extern int prune_empty_dirs;
! 76: +extern int checksum_files;
! 77: extern int copy_links;
! 78: extern int copy_unsafe_links;
! 79: extern int protocol_version;
! 80: @@ -73,6 +76,7 @@ extern int sender_symlink_iconv;
! 81: extern int output_needs_newline;
! 82: extern int sender_keeps_checksum;
! 83: extern int unsort_ndx;
! 84: +extern char *basis_dir[];
! 85: extern uid_t our_uid;
! 86: extern struct stats stats;
! 87: extern char *filesfrom_host;
! 88: @@ -90,6 +94,20 @@ extern int filesfrom_convert;
! 89: extern iconv_t ic_send, ic_recv;
! 90: #endif
! 91:
! 92: +#ifdef HAVE_UTIMENSAT
! 93: +#ifdef HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC
! 94: +#define ST_MTIME_NSEC st_mtim.tv_nsec
! 95: +#elif defined(HAVE_STRUCT_STAT_ST_MTIMENSEC)
! 96: +#define ST_MTIME_NSEC st_mtimensec
! 97: +#endif
! 98: +#endif
! 99: +
! 100: +#define RSYNCSUMS_FILE ".rsyncsums"
! 101: +#define RSYNCSUMS_LEN (sizeof RSYNCSUMS_FILE-1)
! 102: +
! 103: +#define CLEAN_STRIP_ROOT (1<<0)
! 104: +#define CLEAN_KEEP_LAST (1<<1)
! 105: +
! 106: #define PTR_SIZE (sizeof (struct file_struct *))
! 107:
! 108: int io_error;
! 109: @@ -134,8 +152,12 @@ static char empty_sum[MAX_DIGEST_LEN];
! 110: static int flist_count_offset; /* for --delete --progress */
! 111: static int show_filelist_progress;
! 112:
! 113: +static struct csum_cache {
! 114: + struct file_list *flist;
! 115: +} *csum_cache = NULL;
! 116: +
! 117: static struct file_list *flist_new(int flags, const char *msg);
! 118: -static void flist_sort_and_clean(struct file_list *flist, int strip_root);
! 119: +static void flist_sort_and_clean(struct file_list *flist, int flags);
! 120: static void output_flist(struct file_list *flist);
! 121:
! 122: void init_flist(void)
! 123: @@ -324,6 +346,235 @@ static void flist_done_allocating(struct file_list *flist)
! 124: flist->pool_boundary = ptr;
! 125: }
! 126:
! 127: +void reset_checksum_cache()
! 128: +{
! 129: + int slot, slots = am_sender ? 1 : basis_dir_cnt + 1;
! 130: +
! 131: + if (!csum_cache)
! 132: + csum_cache = new_array0(struct csum_cache, slots);
! 133: +
! 134: + for (slot = 0; slot < slots; slot++) {
! 135: + struct file_list *flist = csum_cache[slot].flist;
! 136: +
! 137: + if (flist) {
! 138: + /* Reset the pool memory and empty the file-list array. */
! 139: + pool_free_old(flist->file_pool,
! 140: + pool_boundary(flist->file_pool, 0));
! 141: + flist->used = 0;
! 142: + } else
! 143: + flist = csum_cache[slot].flist = flist_new(FLIST_TEMP, "reset_checksum_cache");
! 144: +
! 145: + flist->low = 0;
! 146: + flist->high = -1;
! 147: + flist->next = NULL;
! 148: + }
! 149: +}
! 150: +
! 151: +/* The basename_len count is the length of the basename + 1 for the '\0'. */
! 152: +static int add_checksum(struct file_list *flist, const char *dirname,
! 153: + const char *basename, int basename_len, OFF_T file_length,
! 154: + time_t mtime, uint32 ctime, uint32 inode,
! 155: + const char *sum)
! 156: +{
! 157: + struct file_struct *file;
! 158: + int alloc_len, extra_len;
! 159: + char *bp;
! 160: +
! 161: + if (basename_len == RSYNCSUMS_LEN+1 && *basename == '.'
! 162: + && strcmp(basename, RSYNCSUMS_FILE) == 0)
! 163: + return 0;
! 164: +
! 165: + /* "2" is for a 32-bit ctime num and an 32-bit inode num. */
! 166: + extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT + 2)
! 167: + * EXTRA_LEN;
! 168: +#if EXTRA_ROUNDING > 0
! 169: + if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
! 170: + extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN;
! 171: +#endif
! 172: + alloc_len = FILE_STRUCT_LEN + extra_len + basename_len;
! 173: + bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum");
! 174: +
! 175: + memset(bp, 0, extra_len + FILE_STRUCT_LEN);
! 176: + bp += extra_len;
! 177: + file = (struct file_struct *)bp;
! 178: + bp += FILE_STRUCT_LEN;
! 179: +
! 180: + memcpy(bp, basename, basename_len);
! 181: +
! 182: + file->mode = S_IFREG;
! 183: + file->modtime = mtime;
! 184: + file->len32 = (uint32)file_length;
! 185: + if (file_length > 0xFFFFFFFFu) {
! 186: + file->flags |= FLAG_LENGTH64;
! 187: + OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32);
! 188: + }
! 189: + file->dirname = dirname;
! 190: + F_CTIME(file) = ctime;
! 191: + F_INODE(file) = inode;
! 192: + bp = F_SUM(file);
! 193: + memcpy(bp, sum, flist_csum_len);
! 194: +
! 195: + flist_expand(flist, 1);
! 196: + flist->files[flist->used++] = file;
! 197: +
! 198: + flist->sorted = flist->files;
! 199: +
! 200: + return 1;
! 201: +}
! 202: +
! 203: +/* The "dirname" arg's data must remain unchanged during the lifespan of
! 204: + * the created csum_cache[].flist object because we use it directly. */
! 205: +static void read_checksums(int slot, struct file_list *flist, const char *dirname)
! 206: +{
! 207: + char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN];
! 208: + FILE *fp;
! 209: + char *cp;
! 210: + int len, i;
! 211: + time_t mtime;
! 212: + OFF_T file_length;
! 213: + uint32 ctime, inode;
! 214: + int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0;
! 215: +
! 216: + if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN))
! 217: + return;
! 218: + if (dlen)
! 219: + fbuf[dlen++] = '/';
! 220: + else
! 221: + dirname = NULL;
! 222: + strlcpy(fbuf+dlen, RSYNCSUMS_FILE, sizeof fbuf - dlen);
! 223: + if (slot) {
! 224: + pathjoin(line, sizeof line, basis_dir[slot-1], fbuf);
! 225: + cp = line;
! 226: + } else
! 227: + cp = fbuf;
! 228: + if (!(fp = fopen(cp, "r")))
! 229: + return;
! 230: +
! 231: + while (fgets(line, sizeof line, fp)) {
! 232: + cp = line;
! 233: + if (checksum_type == 5) {
! 234: + char *alt_sum = cp;
! 235: + if (*cp == '=')
! 236: + while (*++cp == '=') {}
! 237: + else
! 238: + while (isHexDigit(cp)) cp++;
! 239: + if (cp - alt_sum != MD4_DIGEST_LEN*2 || *cp != ' ')
! 240: + break;
! 241: + while (*++cp == ' ') {}
! 242: + }
! 243: +
! 244: + if (*cp == '=') {
! 245: + continue;
! 246: + } else {
! 247: + for (i = 0; i < flist_csum_len*2; i++, cp++) {
! 248: + int x;
! 249: + if (isHexDigit(cp)) {
! 250: + if (isDigit(cp))
! 251: + x = *cp - '0';
! 252: + else
! 253: + x = (*cp & 0xF) + 9;
! 254: + } else {
! 255: + cp = "";
! 256: + break;
! 257: + }
! 258: + if (i & 1)
! 259: + sum[i/2] |= x;
! 260: + else
! 261: + sum[i/2] = x << 4;
! 262: + }
! 263: + }
! 264: + if (*cp != ' ')
! 265: + break;
! 266: + while (*++cp == ' ') {}
! 267: +
! 268: + if (checksum_type != 5) {
! 269: + char *alt_sum = cp;
! 270: + if (*cp == '=')
! 271: + while (*++cp == '=') {}
! 272: + else
! 273: + while (isHexDigit(cp)) cp++;
! 274: + if (cp - alt_sum != MD5_DIGEST_LEN*2 || *cp != ' ')
! 275: + break;
! 276: + while (*++cp == ' ') {}
! 277: + }
! 278: +
! 279: + file_length = 0;
! 280: + while (isDigit(cp))
! 281: + file_length = file_length * 10 + *cp++ - '0';
! 282: + if (*cp != ' ')
! 283: + break;
! 284: + while (*++cp == ' ') {}
! 285: +
! 286: + mtime = 0;
! 287: + while (isDigit(cp))
! 288: + mtime = mtime * 10 + *cp++ - '0';
! 289: + if (*cp != ' ')
! 290: + break;
! 291: + while (*++cp == ' ') {}
! 292: +
! 293: + ctime = 0;
! 294: + while (isDigit(cp))
! 295: + ctime = ctime * 10 + *cp++ - '0';
! 296: + if (*cp != ' ')
! 297: + break;
! 298: + while (*++cp == ' ') {}
! 299: +
! 300: + inode = 0;
! 301: + while (isDigit(cp))
! 302: + inode = inode * 10 + *cp++ - '0';
! 303: + if (*cp != ' ')
! 304: + break;
! 305: + while (*++cp == ' ') {}
! 306: +
! 307: + len = strlen(cp);
! 308: + while (len && (cp[len-1] == '\n' || cp[len-1] == '\r'))
! 309: + len--;
! 310: + if (!len)
! 311: + break;
! 312: + cp[len++] = '\0'; /* len now counts the null */
! 313: + if (strchr(cp, '/'))
! 314: + break;
! 315: + if (len > MAXPATHLEN)
! 316: + continue;
! 317: +
! 318: + strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen);
! 319: +
! 320: + add_checksum(flist, dirname, cp, len, file_length,
! 321: + mtime, ctime, inode,
! 322: + sum);
! 323: + }
! 324: + fclose(fp);
! 325: +
! 326: + flist_sort_and_clean(flist, CLEAN_KEEP_LAST);
! 327: +}
! 328: +
! 329: +void get_cached_checksum(int slot, const char *fname, struct file_struct *file,
! 330: + STRUCT_STAT *stp, char *sum_buf)
! 331: +{
! 332: + struct file_list *flist = csum_cache[slot].flist;
! 333: + int j;
! 334: +
! 335: + if (!flist->next) {
! 336: + flist->next = cur_flist; /* next points from checksum flist to file flist */
! 337: + read_checksums(slot, flist, file->dirname);
! 338: + }
! 339: +
! 340: + if ((j = flist_find(flist, file)) >= 0) {
! 341: + struct file_struct *fp = flist->sorted[j];
! 342: +
! 343: + if (F_LENGTH(fp) == stp->st_size
! 344: + && fp->modtime == stp->st_mtime
! 345: + && (checksum_files & CSF_LAX
! 346: + || (F_CTIME(fp) == (uint32)stp->st_ctime
! 347: + && F_INODE(fp) == (uint32)stp->st_ino))) {
! 348: + memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN);
! 349: + return;
! 350: + }
! 351: + }
! 352: +
! 353: + file_checksum(fname, stp, sum_buf);
! 354: +}
! 355: +
! 356: /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's
! 357: * F_PATHNAME(), or (2) "NULL, dir, dirlen" to chdir() to the supplied dir,
! 358: * with dir == NULL taken to be the starting directory, and dirlen < 0
! 359: @@ -1201,7 +1452,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
! 360: STRUCT_STAT *stp, int flags, int filter_level)
! 361: {
! 362: static char *lastdir;
! 363: - static int lastdir_len = -1;
! 364: + static int lastdir_len = -2;
! 365: struct file_struct *file;
! 366: char thisname[MAXPATHLEN];
! 367: char linkname[MAXPATHLEN];
! 368: @@ -1347,9 +1598,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
! 369: memcpy(lastdir, thisname, len);
! 370: lastdir[len] = '\0';
! 371: lastdir_len = len;
! 372: + if (checksum_files && am_sender && flist)
! 373: + reset_checksum_cache();
! 374: }
! 375: - } else
! 376: + } else {
! 377: basename = thisname;
! 378: + if (checksum_files && am_sender && flist && lastdir_len == -2) {
! 379: + lastdir_len = -1;
! 380: + reset_checksum_cache();
! 381: + }
! 382: + }
! 383: basename_len = strlen(basename) + 1; /* count the '\0' */
! 384:
! 385: #ifdef SUPPORT_LINKS
! 386: @@ -1367,11 +1625,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
! 387: extra_len += EXTRA_LEN;
! 388: #endif
! 389:
! 390: - if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
! 391: - file_checksum(thisname, &st, tmp_sum);
! 392: - if (sender_keeps_checksum)
! 393: - extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
! 394: - }
! 395: + if (sender_keeps_checksum && S_ISREG(st.st_mode))
! 396: + extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
! 397:
! 398: #if EXTRA_ROUNDING > 0
! 399: if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
! 400: @@ -1460,8 +1715,14 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
! 401: return NULL;
! 402: }
! 403:
! 404: - if (sender_keeps_checksum && S_ISREG(st.st_mode))
! 405: - memcpy(F_SUM(file), tmp_sum, flist_csum_len);
! 406: + if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
! 407: + if (flist && checksum_files)
! 408: + get_cached_checksum(0, thisname, file, &st, tmp_sum);
! 409: + else
! 410: + file_checksum(thisname, &st, tmp_sum);
! 411: + if (sender_keeps_checksum)
! 412: + memcpy(F_SUM(file), tmp_sum, flist_csum_len);
! 413: + }
! 414:
! 415: if (unsort_ndx)
! 416: F_NDX(file) = stats.num_dirs;
! 417: @@ -2673,7 +2934,7 @@ struct file_list *recv_file_list(int f, int dir_ndx)
! 418: /* The --relative option sends paths with a leading slash, so we need
! 419: * to specify the strip_root option here. We rejected leading slashes
! 420: * for a non-relative transfer in recv_file_entry(). */
! 421: - flist_sort_and_clean(flist, relative_paths);
! 422: + flist_sort_and_clean(flist, relative_paths ? CLEAN_STRIP_ROOT : 0);
! 423:
! 424: if (protocol_version < 30) {
! 425: /* Recv the io_error flag */
! 426: @@ -2918,7 +3179,7 @@ void flist_free(struct file_list *flist)
! 427:
! 428: /* This routine ensures we don't have any duplicate names in our file list.
! 429: * duplicate names can cause corruption because of the pipelining. */
! 430: -static void flist_sort_and_clean(struct file_list *flist, int strip_root)
! 431: +static void flist_sort_and_clean(struct file_list *flist, int flags)
! 432: {
! 433: char fbuf[MAXPATHLEN];
! 434: int i, prev_i;
! 435: @@ -2969,7 +3230,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root)
! 436: /* If one is a dir and the other is not, we want to
! 437: * keep the dir because it might have contents in the
! 438: * list. Otherwise keep the first one. */
! 439: - if (S_ISDIR(file->mode)) {
! 440: + if (S_ISDIR(file->mode) || flags & CLEAN_KEEP_LAST) {
! 441: struct file_struct *fp = flist->sorted[j];
! 442: if (!S_ISDIR(fp->mode))
! 443: keep = i, drop = j;
! 444: @@ -2985,8 +3246,8 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root)
! 445: } else
! 446: keep = j, drop = i;
! 447:
! 448: - if (!am_sender) {
! 449: - if (DEBUG_GTE(DUP, 1)) {
! 450: + if (!am_sender || flags & CLEAN_KEEP_LAST) {
! 451: + if (DEBUG_GTE(DUP, 1) && !(flags & CLEAN_KEEP_LAST)) {
! 452: rprintf(FINFO,
! 453: "removing duplicate name %s from file list (%d)\n",
! 454: f_name(file, fbuf), drop + flist->ndx_start);
! 455: @@ -3008,7 +3269,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root)
! 456: }
! 457: flist->high = prev_i;
! 458:
! 459: - if (strip_root) {
! 460: + if (flags & CLEAN_STRIP_ROOT) {
! 461: /* We need to strip off the leading slashes for relative
! 462: * paths, but this must be done _after_ the sorting phase. */
! 463: for (i = flist->low; i <= flist->high; i++) {
! 464: diff --git a/generator.c b/generator.c
! 465: --- a/generator.c
! 466: +++ b/generator.c
! 467: @@ -52,6 +52,7 @@ extern int delete_after;
! 468: extern int missing_args;
! 469: extern int msgdone_cnt;
! 470: extern int ignore_errors;
! 471: +extern int checksum_files;
! 472: extern int remove_source_files;
! 473: extern int delay_updates;
! 474: extern int update_only;
! 475: @@ -601,7 +602,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre
! 476:
! 477:
! 478: /* Perform our quick-check heuristic for determining if a file is unchanged. */
! 479: -int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
! 480: +int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st, int slot)
! 481: {
! 482: if (st->st_size != F_LENGTH(file))
! 483: return 0;
! 484: @@ -610,7 +611,10 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
! 485: of the file time to determine whether to sync */
! 486: if (always_checksum > 0 && S_ISREG(st->st_mode)) {
! 487: char sum[MAX_DIGEST_LEN];
! 488: - file_checksum(fn, st, sum);
! 489: + if (checksum_files && slot >= 0)
! 490: + get_cached_checksum(slot, fn, file, st, sum);
! 491: + else
! 492: + file_checksum(fn, st, sum);
! 493: return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
! 494: }
! 495:
! 496: @@ -907,7 +911,7 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx,
! 497: best_match = j;
! 498: match_level = 1;
! 499: }
! 500: - if (!unchanged_file(cmpbuf, file, &sxp->st))
! 501: + if (!unchanged_file(cmpbuf, file, &sxp->st, j+1))
! 502: continue;
! 503: if (match_level == 1) {
! 504: best_match = j;
! 505: @@ -1220,7 +1224,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
! 506: * --ignore-non-existing, daemon exclude, or mkdir failure. */
! 507: static struct file_struct *skip_dir = NULL;
! 508: static struct file_list *fuzzy_dirlist[MAX_BASIS_DIRS+1];
! 509: - static int need_fuzzy_dirlist = 0;
! 510: + static int need_new_dirscan = 0;
! 511: struct file_struct *fuzzy_file = NULL;
! 512: int fd = -1, f_copy = -1;
! 513: stat_x sx, real_sx;
! 514: @@ -1337,8 +1341,9 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
! 515: fuzzy_dirlist[i] = NULL;
! 516: }
! 517: }
! 518: - need_fuzzy_dirlist = 1;
! 519: - }
! 520: + need_new_dirscan = 1;
! 521: + } else if (checksum_files)
! 522: + need_new_dirscan = 1;
! 523: #ifdef SUPPORT_ACLS
! 524: if (!preserve_perms)
! 525: dflt_perms = default_perms_for_dir(dn);
! 526: @@ -1346,6 +1351,24 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
! 527: }
! 528: parent_dirname = dn;
! 529:
! 530: + if (need_new_dirscan && S_ISREG(file->mode)) {
! 531: + int i;
! 532: + strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf);
! 533: + for (i = 0; i < fuzzy_basis; i++) {
! 534: + if (i && pathjoin(fnamecmpbuf, MAXPATHLEN, basis_dir[i-1], dn) >= MAXPATHLEN)
! 535: + continue;
! 536: + fuzzy_dirlist[i] = get_dirlist(fnamecmpbuf, -1, GDL_IGNORE_FILTER_RULES | GDL_PERHAPS_DIR);
! 537: + if (fuzzy_dirlist[i] && fuzzy_dirlist[i]->used == 0) {
! 538: + flist_free(fuzzy_dirlist[i]);
! 539: + fuzzy_dirlist[i] = NULL;
! 540: + }
! 541: + }
! 542: + if (checksum_files) {
! 543: + reset_checksum_cache();
! 544: + }
! 545: + need_new_dirscan = 0;
! 546: + }
! 547: +
! 548: statret = link_stat(fname, &sx.st, keep_dirlinks && is_dir);
! 549: stat_errno = errno;
! 550: }
! 551: @@ -1749,22 +1772,6 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
! 552: partialptr = NULL;
! 553:
! 554: if (statret != 0 && fuzzy_basis) {
! 555: - if (need_fuzzy_dirlist && S_ISREG(file->mode)) {
! 556: - const char *dn = file->dirname ? file->dirname : ".";
! 557: - int i;
! 558: - strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf);
! 559: - for (i = 0; i < fuzzy_basis; i++) {
! 560: - if (i && pathjoin(fnamecmpbuf, MAXPATHLEN, basis_dir[i-1], dn) >= MAXPATHLEN)
! 561: - continue;
! 562: - fuzzy_dirlist[i] = get_dirlist(fnamecmpbuf, -1, GDL_IGNORE_FILTER_RULES | GDL_PERHAPS_DIR);
! 563: - if (fuzzy_dirlist[i] && fuzzy_dirlist[i]->used == 0) {
! 564: - flist_free(fuzzy_dirlist[i]);
! 565: - fuzzy_dirlist[i] = NULL;
! 566: - }
! 567: - }
! 568: - need_fuzzy_dirlist = 0;
! 569: - }
! 570: -
! 571: /* Sets fnamecmp_type to FNAMECMP_FUZZY or above. */
! 572: fuzzy_file = find_fuzzy(file, fuzzy_dirlist, &fnamecmp_type);
! 573: if (fuzzy_file) {
! 574: @@ -1797,7 +1804,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
! 575: ;
! 576: else if (fnamecmp_type >= FNAMECMP_FUZZY)
! 577: ;
! 578: - else if (unchanged_file(fnamecmp, file, &sx.st)) {
! 579: + else if (unchanged_file(fnamecmp, file, &sx.st, fnamecmp_type == FNAMECMP_FNAME ? 0 : -1)) {
! 580: if (partialptr) {
! 581: do_unlink(partialptr);
! 582: handle_partial_dir(partialptr, PDIR_DELETE);
! 583: diff --git a/hlink.c b/hlink.c
! 584: --- a/hlink.c
! 585: +++ b/hlink.c
! 586: @@ -406,7 +406,7 @@ int hard_link_check(struct file_struct *file, int ndx, char *fname,
! 587: }
! 588: break;
! 589: }
! 590: - if (!unchanged_file(cmpbuf, file, &alt_sx.st))
! 591: + if (!unchanged_file(cmpbuf, file, &alt_sx.st, j+1))
! 592: continue;
! 593: statret = 1;
! 594: if (unchanged_attrs(cmpbuf, file, &alt_sx))
! 595: diff --git a/loadparm.c b/loadparm.c
! 596: --- a/loadparm.c
! 597: +++ b/loadparm.c
! 598: @@ -162,6 +162,13 @@ static struct enum_list enum_syslog_facility[] = {
! 599: { -1, NULL }
! 600: };
! 601:
! 602: +static struct enum_list enum_checksum_files[] = {
! 603: + { CSF_IGNORE_FILES, "none" },
! 604: + { CSF_LAX_MODE, "lax" },
! 605: + { CSF_STRICT_MODE, "strict" },
! 606: + { -1, NULL }
! 607: +};
! 608: +
! 609: /* Expand %VAR% references. Any unknown vars or unrecognized
! 610: * syntax leaves the raw chars unchanged. */
! 611: static char *expand_vars(const char *str)
! 612: diff --git a/options.c b/options.c
! 613: --- a/options.c
! 614: +++ b/options.c
! 615: @@ -117,6 +117,7 @@ size_t bwlimit_writemax = 0;
! 616: int ignore_existing = 0;
! 617: int ignore_non_existing = 0;
! 618: int need_messages_from_generator = 0;
! 619: +int checksum_files = CSF_IGNORE_FILES;
! 620: int max_delete = INT_MIN;
! 621: OFF_T max_size = -1;
! 622: OFF_T min_size = -1;
! 623: @@ -573,7 +574,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
! 624: OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
! 625: OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
! 626: OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE,
! 627: - OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR,
! 628: + OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, OPT_SUMFILES,
! 629: OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS,
! 630: OPT_STOP_AFTER, OPT_STOP_AT,
! 631: OPT_REFUSED_BASE = 9000};
! 632: @@ -729,6 +730,7 @@ static struct poptOption long_options[] = {
! 633: {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
! 634: {"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
! 635: {"cc", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
! 636: + {"sumfiles", 0, POPT_ARG_STRING, 0, OPT_SUMFILES, 0, 0 },
! 637: {"block-size", 'B', POPT_ARG_STRING, 0, OPT_BLOCK_SIZE, 0, 0 },
! 638: {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
! 639: {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
! 640: @@ -1722,6 +1724,23 @@ int parse_arguments(int *argc_p, const char ***argv_p)
! 641: }
! 642: break;
! 643:
! 644: + case OPT_SUMFILES:
! 645: + arg = poptGetOptArg(pc);
! 646: + checksum_files = 0;
! 647: + if (strcmp(arg, "lax") == 0)
! 648: + checksum_files |= CSF_LAX_MODE;
! 649: + else if (strcmp(arg, "strict") == 0)
! 650: + checksum_files |= CSF_STRICT_MODE;
! 651: + else if (strcmp(arg, "none") == 0)
! 652: + checksum_files = CSF_IGNORE_FILES;
! 653: + else {
! 654: + snprintf(err_buf, sizeof err_buf,
! 655: + "Invalid argument passed to --sumfiles (%s)\n",
! 656: + arg);
! 657: + return 0;
! 658: + }
! 659: + break;
! 660: +
! 661: case OPT_INFO:
! 662: arg = poptGetOptArg(pc);
! 663: parse_output_words(info_words, info_levels, arg, USER_PRIORITY);
! 664: @@ -2052,6 +2071,9 @@ int parse_arguments(int *argc_p, const char ***argv_p)
! 665: }
! 666: #endif
! 667:
! 668: + if (!always_checksum)
! 669: + checksum_files = CSF_IGNORE_FILES;
! 670: +
! 671: if (write_batch && read_batch) {
! 672: snprintf(err_buf, sizeof err_buf,
! 673: "--write-batch and --read-batch can not be used together\n");
! 674: diff --git a/rsync.1.md b/rsync.1.md
! 675: --- a/rsync.1.md
! 676: +++ b/rsync.1.md
! 677: @@ -338,6 +338,7 @@ detailed description below for a complete description.
! 678: --quiet, -q suppress non-error messages
! 679: --no-motd suppress daemon-mode MOTD
! 680: --checksum, -c skip based on checksum, not mod-time & size
! 681: +--sumfiles=MODE use .rsyncsums to speedup --checksum mode
! 682: --archive, -a archive mode; equals -rlptgoD (no -H,-A,-X)
! 683: --no-OPTION turn off an implied OPTION (e.g. --no-D)
! 684: --recursive, -r recurse into directories
! 685: @@ -698,6 +699,8 @@ your home directory (remove the '=' for that).
! 686: file that has the same size as the corresponding sender's file: files with
! 687: either a changed size or a changed checksum are selected for transfer.
! 688:
! 689: + See also the `--sumfiles` option for a way to use cached checksum data.
! 690: +
! 691: Note that rsync always verifies that each _transferred_ file was correctly
! 692: reconstructed on the receiving side by checking a whole-file checksum that
! 693: is generated as the file is transferred, but that automatic
! 694: @@ -708,6 +711,38 @@ your home directory (remove the '=' for that).
! 695: can be overridden using either the `--checksum-choice` (`--cc`) option or an
! 696: environment variable that is discussed in that option's section.
! 697:
! 698: +0. `--sumfiles=MODE`
! 699: +
! 700: + This option tells rsync to make use of any cached checksum information it
! 701: + finds in per-directory .rsyncsums files when the current transfer is using
! 702: + the `--checksum` option. If the checksum data is up-to-date, it is used
! 703: + instead of recomputing it, saving both disk I/O and CPU time. If the
! 704: + checksum data is missing or outdated, the checksum is computed just as it
! 705: + would be if `--sumfiles` was not specified.
! 706: +
! 707: + The MODE value is either "lax", for relaxed checking (which compares size
! 708: + and mtime), "strict" (which also compares ctime and inode), or "none" to
! 709: + ignore any .rsyncsums files ("none" is the default). Rsync does not create
! 710: + or update these files, but there is a perl script in the support directory
! 711: + named "rsyncsums" that can be used for that.
! 712: +
! 713: + This option has no effect unless `--checksum`, `-c` was also specified. It
! 714: + also only affects the current side of the transfer, so if you want the
! 715: + remote side to parse its own .rsyncsums files, specify the option via
! 716: + `--remote-option` (`-M`) (e.g. "`-M--sumfiles=lax`").
! 717: +
! 718: + To avoid transferring the system's checksum files, you can use an exclude
! 719: + (e.g. `--exclude=.rsyncsums`). To make this easier to type, you can use a
! 720: + popt alias. For instance, adding the following line in your ~/.popt file
! 721: + defines a `--cs` option that enables lax checksum files and excludes the
! 722: + checksum files:
! 723: +
! 724: + > rsync alias --cs -c --sumfiles=lax -M--sumfiles=lax -f-_.rsyncsums
! 725: +
! 726: + An rsync daemon does not allow the client to control this setting, so see
! 727: + the "checksum files" daemon parameter for information on how to make a
! 728: + daemon use cached checksum data.
! 729: +
! 730: 0. `--archive`, `-a`
! 731:
! 732: This is equivalent to `-rlptgoD`. It is a quick way of saying you want
! 733: diff --git a/rsync.h b/rsync.h
! 734: --- a/rsync.h
! 735: +++ b/rsync.h
! 736: @@ -882,6 +882,10 @@ extern int xattrs_ndx;
! 737: #define F_SUM(f) ((char*)OPT_EXTRA(f, START_BUMP(f) + HLINK_BUMP(f) \
! 738: + SUM_EXTRA_CNT - 1))
! 739:
! 740: +/* These are only valid on an entry derived from a checksum file. */
! 741: +#define F_CTIME(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT)->unum
! 742: +#define F_INODE(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT + 1)->unum
! 743: +
! 744: /* Some utility defines: */
! 745: #define F_IS_ACTIVE(f) (f)->basename[0]
! 746: #define F_IS_HLINKED(f) ((f)->flags & FLAG_HLINKED)
! 747: @@ -1094,6 +1098,13 @@ typedef struct {
! 748: #define RELNAMECACHE_LEN (offsetof(relnamecache, fname))
! 749: #endif
! 750:
! 751: +#define CSF_ENABLE (1<<1)
! 752: +#define CSF_LAX (1<<2)
! 753: +
! 754: +#define CSF_IGNORE_FILES 0
! 755: +#define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX)
! 756: +#define CSF_STRICT_MODE (CSF_ENABLE)
! 757: +
! 758: #include "byteorder.h"
! 759: #include "lib/mdigest.h"
! 760: #include "lib/wildmatch.h"
! 761: diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md
! 762: --- a/rsyncd.conf.5.md
! 763: +++ b/rsyncd.conf.5.md
! 764: @@ -419,6 +419,19 @@ the values of parameters. See the GLOBAL PARAMETERS section for more details.
! 765: the max connections limit is not exceeded for the modules sharing the lock
! 766: file. The default is `/var/run/rsyncd.lock`.
! 767:
! 768: +0. `checksum files`
! 769: +
! 770: + This parameter tells rsync to make use of any cached checksum information
! 771: + it finds in per-directory .rsyncsums files when the current transfer is
! 772: + using the `--checksum` option. The value can be set to either "lax",
! 773: + "strict", or "none". See the client's `--sumfiles` option for what these
! 774: + choices do.
! 775: +
! 776: + Note also that the client's command-line option, `--sumfiles`, has no
! 777: + effect on a daemon. A daemon will only access checksum files if this
! 778: + config option tells it to. See also the `exclude` directive for a way to
! 779: + hide the .rsyncsums files from the user.
! 780: +
! 781: 0. `read only`
! 782:
! 783: This parameter determines whether clients will be able to upload files or
! 784: diff --git a/support/rsyncsums b/support/rsyncsums
! 785: new file mode 100755
! 786: --- /dev/null
! 787: +++ b/support/rsyncsums
! 788: @@ -0,0 +1,201 @@
! 789: +#!/usr/bin/perl -w
! 790: +use strict;
! 791: +
! 792: +use Getopt::Long;
! 793: +use Cwd qw(abs_path cwd);
! 794: +use Digest::MD4;
! 795: +use Digest::MD5;
! 796: +
! 797: +our $SUMS_FILE = '.rsyncsums';
! 798: +
! 799: +&Getopt::Long::Configure('bundling');
! 800: +&usage if !&GetOptions(
! 801: + 'recurse|r' => \( my $recurse_opt ),
! 802: + 'mode|m=s' => \( my $cmp_mode = 'strict' ),
! 803: + 'check|c' => \( my $check_opt ),
! 804: + 'verbose|v+' => \( my $verbosity = 0 ),
! 805: + 'help|h' => \( my $help_opt ),
! 806: +);
! 807: +&usage if $help_opt || $cmp_mode !~ /^(lax|strict)$/;
! 808: +
! 809: +my $ignore_ctime_and_inode = $cmp_mode eq 'lax' ? 0 : 1;
! 810: +
! 811: +my $start_dir = cwd();
! 812: +
! 813: +my @dirs = @ARGV;
! 814: +@dirs = '.' unless @dirs;
! 815: +foreach (@dirs) {
! 816: + $_ = abs_path($_);
! 817: +}
! 818: +
! 819: +$| = 1;
! 820: +
! 821: +my $exit_code = 0;
! 822: +
! 823: +my $md4 = Digest::MD4->new;
! 824: +my $md5 = Digest::MD5->new;
! 825: +
! 826: +while (@dirs) {
! 827: + my $dir = shift @dirs;
! 828: +
! 829: + if (!chdir($dir)) {
! 830: + warn "Unable to chdir to $dir: $!\n";
! 831: + next;
! 832: + }
! 833: + if (!opendir(DP, '.')) {
! 834: + warn "Unable to opendir $dir: $!\n";
! 835: + next;
! 836: + }
! 837: +
! 838: + my $reldir = $dir;
! 839: + $reldir =~ s#^$start_dir(/|$)# $1 ? '' : '.' #eo;
! 840: + if ($verbosity) {
! 841: + print "$reldir ... ";
! 842: + print "\n" if $check_opt;
! 843: + }
! 844: +
! 845: + my %cache;
! 846: + my $f_cnt = 0;
! 847: + if (open(FP, '<', $SUMS_FILE)) {
! 848: + while (<FP>) {
! 849: + chomp;
! 850: + my($sum4, $sum5, $size, $mtime, $ctime, $inode, $fn) = split(' ', $_, 7);
! 851: + $cache{$fn} = [ 0, $sum4, $sum5, $size, $mtime, $ctime & 0xFFFFFFFF, $inode & 0xFFFFFFFF ];
! 852: + $f_cnt++;
! 853: + }
! 854: + close FP;
! 855: + }
! 856: +
! 857: + my @subdirs;
! 858: + my $d_cnt = 0;
! 859: + my $update_cnt = 0;
! 860: + while (defined(my $fn = readdir(DP))) {
! 861: + next if $fn =~ /^\.\.?$/ || $fn =~ /^\Q$SUMS_FILE\E$/o || -l $fn;
! 862: + if (-d _) {
! 863: + push(@subdirs, "$dir/$fn") unless $fn =~ /^(CVS|\.svn|\.git|\.bzr)$/;
! 864: + next;
! 865: + }
! 866: + next unless -f _;
! 867: +
! 868: + my($size,$mtime,$ctime,$inode) = (stat(_))[7,9,10,1];
! 869: + $ctime &= 0xFFFFFFFF;
! 870: + $inode &= 0xFFFFFFFF;
! 871: + my $ref = $cache{$fn};
! 872: + $d_cnt++;
! 873: +
! 874: + if (!$check_opt) {
! 875: + if (defined $ref) {
! 876: + $$ref[0] = 1;
! 877: + if ($$ref[3] == $size
! 878: + && $$ref[4] == $mtime
! 879: + && ($ignore_ctime_and_inode || ($$ref[5] == $ctime && $$ref[6] == $inode))
! 880: + && $$ref[1] !~ /=/ && $$ref[2] !~ /=/) {
! 881: + next;
! 882: + }
! 883: + }
! 884: + if (!$update_cnt++) {
! 885: + print "UPDATING\n" if $verbosity;
! 886: + }
! 887: + }
! 888: +
! 889: + if (!open(IN, $fn)) {
! 890: + print STDERR "Unable to read $fn: $!\n";
! 891: + if (defined $ref) {
! 892: + delete $cache{$fn};
! 893: + $f_cnt--;
! 894: + }
! 895: + next;
! 896: + }
! 897: +
! 898: + my($sum4, $sum5);
! 899: + while (1) {
! 900: + while (sysread(IN, $_, 64*1024)) {
! 901: + $md4->add($_);
! 902: + $md5->add($_);
! 903: + }
! 904: + $sum4 = $md4->hexdigest;
! 905: + $sum5 = $md5->hexdigest;
! 906: + print " $sum4 $sum5" if $verbosity > 2;
! 907: + print " $fn" if $verbosity > 1;
! 908: + my($size2,$mtime2,$ctime2,$inode2) = (stat(IN))[7,9,10,1];
! 909: + $ctime2 &= 0xFFFFFFFF;
! 910: + $inode2 &= 0xFFFFFFFF;
! 911: + last if $size == $size2 && $mtime == $mtime2
! 912: + && ($ignore_ctime_and_inode || ($ctime == $ctime2 && $inode == $inode2));
! 913: + $size = $size2;
! 914: + $mtime = $mtime2;
! 915: + $ctime = $ctime2;
! 916: + $inode = $inode2;
! 917: + sysseek(IN, 0, 0);
! 918: + print " REREADING\n" if $verbosity > 1;
! 919: + }
! 920: +
! 921: + close IN;
! 922: +
! 923: + if ($check_opt) {
! 924: + my $dif;
! 925: + if (!defined $ref) {
! 926: + $dif = 'MISSING';
! 927: + } elsif ($sum4 ne $$ref[1] || $sum5 ne $$ref[2]) {
! 928: + $dif = 'FAILED';
! 929: + } else {
! 930: + print " OK\n" if $verbosity > 1;
! 931: + next;
! 932: + }
! 933: + if ($verbosity < 2) {
! 934: + print $verbosity ? ' ' : "$reldir/";
! 935: + print $fn;
! 936: + }
! 937: + print " $dif\n";
! 938: + $exit_code = 1;
! 939: + } else {
! 940: + print "\n" if $verbosity > 1;
! 941: + $cache{$fn} = [ 1, $sum4, $sum5, $size, $mtime, $ctime, $inode ];
! 942: + }
! 943: + }
! 944: +
! 945: + closedir DP;
! 946: +
! 947: + unshift(@dirs, sort @subdirs) if $recurse_opt;
! 948: +
! 949: + if ($check_opt) {
! 950: + ;
! 951: + } elsif ($d_cnt == 0) {
! 952: + if ($f_cnt) {
! 953: + print "(removed $SUMS_FILE) " if $verbosity;
! 954: + unlink($SUMS_FILE);
! 955: + }
! 956: + print "empty\n" if $verbosity;
! 957: + } elsif ($update_cnt || $d_cnt != $f_cnt) {
! 958: + print "UPDATING\n" if $verbosity && !$update_cnt;
! 959: + open(FP, '>', $SUMS_FILE) or die "Unable to write $dir/$SUMS_FILE: $!\n";
! 960: +
! 961: + foreach my $fn (sort keys %cache) {
! 962: + my $ref = $cache{$fn};
! 963: + my($found, $sum4, $sum5, $size, $mtime, $ctime, $inode) = @$ref;
! 964: + next unless $found;
! 965: + printf FP '%s %s %10d %10d %10d %10d %s' . "\n", $sum4, $sum5, $size, $mtime, $ctime, $inode, $fn;
! 966: + }
! 967: + close FP;
! 968: + } else {
! 969: + print "ok\n" if $verbosity;
! 970: + }
! 971: +}
! 972: +
! 973: +exit $exit_code;
! 974: +
! 975: +sub usage
! 976: +{
! 977: + die <<EOT;
! 978: +Usage: rsyncsums [OPTIONS] [DIRS]
! 979: +
! 980: +Options:
! 981: + -r, --recurse Update $SUMS_FILE files in subdirectories too.
! 982: + -m, --mode=MODE Compare entries in either "lax" or "strict" mode. Using
! 983: + "lax" compares size and mtime, while "strict" additionally
! 984: + compares ctime and inode. Default: strict.
! 985: + -c, --check Check if the checksums are right (doesn't update).
! 986: + -v, --verbose Mention what we're doing. Repeat for more info.
! 987: + -h, --help Display this help message.
! 988: +EOT
! 989: +}
! 990: diff -Nurp a/rsync.1 b/rsync.1
! 991: --- a/rsync.1
! 992: +++ b/rsync.1
! 993: @@ -414,6 +414,7 @@ detailed description below for a complet
! 994: --quiet, -q suppress non-error messages
! 995: --no-motd suppress daemon-mode MOTD
! 996: --checksum, -c skip based on checksum, not mod-time & size
! 997: +--sumfiles=MODE use .rsyncsums to speedup --checksum mode
! 998: --archive, -a archive mode; equals -rlptgoD (no -H,-A,-X)
! 999: --no-OPTION turn off an implied OPTION (e.g. --no-D)
! 1000: --recursive, -r recurse into directories
! 1001: @@ -766,6 +767,8 @@ its checksums when it is scanning for ch
! 1002: file that has the same size as the corresponding sender's file: files with
! 1003: either a changed size or a changed checksum are selected for transfer.
! 1004: .IP
! 1005: +See also the \fB\-\-sumfiles\fP option for a way to use cached checksum data.
! 1006: +.IP
! 1007: Note that rsync always verifies that each \fItransferred\fP file was correctly
! 1008: reconstructed on the receiving side by checking a whole-file checksum that
! 1009: is generated as the file is transferred, but that automatic
! 1010: @@ -775,6 +778,40 @@ before-the-transfer "Does this file need
! 1011: The checksum used is auto-negotiated between the client and the server, but
! 1012: can be overridden using either the \fB\-\-checksum-choice\fP (\fB\-\-cc\fP) option or an
! 1013: environment variable that is discussed in that option's section.
! 1014: +.IP "\fB\-\-sumfiles=MODE\fP"
! 1015: +This option tells rsync to make use of any cached checksum information it
! 1016: +finds in per-directory .rsyncsums files when the current transfer is using
! 1017: +the \fB\-\-checksum\fP option. If the checksum data is up-to-date, it is used
! 1018: +instead of recomputing it, saving both disk I/O and CPU time. If the
! 1019: +checksum data is missing or outdated, the checksum is computed just as it
! 1020: +would be if \fB\-\-sumfiles\fP was not specified.
! 1021: +.IP
! 1022: +The MODE value is either "lax", for relaxed checking (which compares size
! 1023: +and mtime), "strict" (which also compares ctime and inode), or "none" to
! 1024: +ignore any .rsyncsums files ("none" is the default). Rsync does not create
! 1025: +or update these files, but there is a perl script in the support directory
! 1026: +named "rsyncsums" that can be used for that.
! 1027: +.IP
! 1028: +This option has no effect unless \fB\-\-checksum\fP, \fB\-c\fP was also specified. It
! 1029: +also only affects the current side of the transfer, so if you want the
! 1030: +remote side to parse its own .rsyncsums files, specify the option via
! 1031: +\fB\-\-remote-option\fP (\fB\-M\fP) (e.g. "\fB\-M\-\-sumfiles=lax\fP").
! 1032: +.IP
! 1033: +To avoid transferring the system's checksum files, you can use an exclude
! 1034: +(e.g. \fB\-\-exclude=.rsyncsums\fP). To make this easier to type, you can use a
! 1035: +popt alias. For instance, adding the following line in your ~/.popt file
! 1036: +defines a \fB\-\-cs\fP option that enables lax checksum files and excludes the
! 1037: +checksum files:
! 1038: +.RS 4
! 1039: +.IP
! 1040: +.nf
! 1041: +rsync alias --cs -c --sumfiles=lax -M--sumfiles=lax -f-_.rsyncsums
! 1042: +.fi
! 1043: +.RE
! 1044: +.IP
! 1045: +An rsync daemon does not allow the client to control this setting, so see
! 1046: +the "checksum files" daemon parameter for information on how to make a
! 1047: +daemon use cached checksum data.
! 1048: .IP "\fB\-\-archive\fP, \fB\-a\fP"
! 1049: This is equivalent to \fB\-rlptgoD\fP. It is a quick way of saying you want
! 1050: recursion and want to preserve almost everything (with \fB\-H\fP being a notable
! 1051: diff -Nurp a/rsync.1.html b/rsync.1.html
! 1052: --- a/rsync.1.html
! 1053: +++ b/rsync.1.html
! 1054: @@ -329,6 +329,7 @@ detailed description below for a complet
! 1055: --quiet, -q suppress non-error messages
! 1056: --no-motd suppress daemon-mode MOTD
! 1057: --checksum, -c skip based on checksum, not mod-time & size
! 1058: +--sumfiles=MODE use .rsyncsums to speedup --checksum mode
! 1059: --archive, -a archive mode; equals -rlptgoD (no -H,-A,-X)
! 1060: --no-OPTION turn off an implied OPTION (e.g. --no-D)
! 1061: --recursive, -r recurse into directories
! 1062: @@ -681,6 +682,7 @@ scan that builds the list of the availab
! 1063: its checksums when it is scanning for changed files, and will checksum any
! 1064: file that has the same size as the corresponding sender's file: files with
! 1065: either a changed size or a changed checksum are selected for transfer.</p>
! 1066: +<p>See also the <code>--sumfiles</code> option for a way to use cached checksum data.</p>
! 1067: <p>Note that rsync always verifies that each <u>transferred</u> file was correctly
! 1068: reconstructed on the receiving side by checking a whole-file checksum that
! 1069: is generated as the file is transferred, but that automatic
! 1070: @@ -691,6 +693,36 @@ can be overridden using either the <code
! 1071: environment variable that is discussed in that option's section.</p>
! 1072: </dd>
! 1073:
! 1074: +<dt><code>--sumfiles=MODE</code></dt><dd>
! 1075: +<p>This option tells rsync to make use of any cached checksum information it
! 1076: +finds in per-directory .rsyncsums files when the current transfer is using
! 1077: +the <code>--checksum</code> option. If the checksum data is up-to-date, it is used
! 1078: +instead of recomputing it, saving both disk I/O and CPU time. If the
! 1079: +checksum data is missing or outdated, the checksum is computed just as it
! 1080: +would be if <code>--sumfiles</code> was not specified.</p>
! 1081: +<p>The MODE value is either "lax", for relaxed checking (which compares size
! 1082: +and mtime), "strict" (which also compares ctime and inode), or "none" to
! 1083: +ignore any .rsyncsums files ("none" is the default). Rsync does not create
! 1084: +or update these files, but there is a perl script in the support directory
! 1085: +named "rsyncsums" that can be used for that.</p>
! 1086: +<p>This option has no effect unless <code>--checksum</code>, <code>-c</code> was also specified. It
! 1087: +also only affects the current side of the transfer, so if you want the
! 1088: +remote side to parse its own .rsyncsums files, specify the option via
! 1089: +<code>--remote-option</code> (<code>-M</code>) (e.g. "<code>-M--sumfiles=lax</code>").</p>
! 1090: +<p>To avoid transferring the system's checksum files, you can use an exclude
! 1091: +(e.g. <code>--exclude=.rsyncsums</code>). To make this easier to type, you can use a
! 1092: +popt alias. For instance, adding the following line in your ~/.popt file
! 1093: +defines a <code>--cs</code> option that enables lax checksum files and excludes the
! 1094: +checksum files:</p>
! 1095: +<blockquote>
! 1096: +<pre><code>rsync alias --cs -c --sumfiles=lax -M--sumfiles=lax -f-_.rsyncsums
! 1097: +</code></pre>
! 1098: +</blockquote>
! 1099: +<p>An rsync daemon does not allow the client to control this setting, so see
! 1100: +the "checksum files" daemon parameter for information on how to make a
! 1101: +daemon use cached checksum data.</p>
! 1102: +</dd>
! 1103: +
! 1104: <dt><code>--archive</code>, <code>-a</code></dt><dd>
! 1105: <p>This is equivalent to <code>-rlptgoD</code>. It is a quick way of saying you want
! 1106: recursion and want to preserve almost everything (with <code>-H</code> being a notable
! 1107: diff -Nurp a/rsyncd.conf.5 b/rsyncd.conf.5
! 1108: --- a/rsyncd.conf.5
! 1109: +++ b/rsyncd.conf.5
! 1110: @@ -394,6 +394,17 @@ This parameter specifies the file to use
! 1111: parameter. The rsync daemon uses record locking on this file to ensure that
! 1112: the max connections limit is not exceeded for the modules sharing the lock
! 1113: file. The default is \fB/var/run/rsyncd.lock\fP.
! 1114: +.IP "\fBchecksum\ files\fP"
! 1115: +This parameter tells rsync to make use of any cached checksum information
! 1116: +it finds in per-directory .rsyncsums files when the current transfer is
! 1117: +using the \fB\-\-checksum\fP option. The value can be set to either "lax",
! 1118: +"strict", or "none". See the client's \fB\-\-sumfiles\fP option for what these
! 1119: +choices do.
! 1120: +.IP
! 1121: +Note also that the client's command-line option, \fB\-\-sumfiles\fP, has no
! 1122: +effect on a daemon. A daemon will only access checksum files if this
! 1123: +config option tells it to. See also the \fBexclude\fP directive for a way to
! 1124: +hide the .rsyncsums files from the user.
! 1125: .IP "\fBread\ only\fP"
! 1126: This parameter determines whether clients will be able to upload files or
! 1127: not. If "read only" is true then any attempted uploads will fail. If
! 1128: diff -Nurp a/rsyncd.conf.5.html b/rsyncd.conf.5.html
! 1129: --- a/rsyncd.conf.5.html
! 1130: +++ b/rsyncd.conf.5.html
! 1131: @@ -405,6 +405,18 @@ the max connections limit is not exceede
! 1132: file. The default is <code>/var/run/rsyncd.lock</code>.</p>
! 1133: </dd>
! 1134:
! 1135: +<dt><code>checksum files</code></dt><dd>
! 1136: +<p>This parameter tells rsync to make use of any cached checksum information
! 1137: +it finds in per-directory .rsyncsums files when the current transfer is
! 1138: +using the <code>--checksum</code> option. The value can be set to either "lax",
! 1139: +"strict", or "none". See the client's <code>--sumfiles</code> option for what these
! 1140: +choices do.</p>
! 1141: +<p>Note also that the client's command-line option, <code>--sumfiles</code>, has no
! 1142: +effect on a daemon. A daemon will only access checksum files if this
! 1143: +config option tells it to. See also the <code>exclude</code> directive for a way to
! 1144: +hide the .rsyncsums files from the user.</p>
! 1145: +</dd>
! 1146: +
! 1147: <dt><code>read only</code></dt><dd>
! 1148: <p>This parameter determines whether clients will be able to upload files or
! 1149: not. If "read only" is true then any attempted uploads will fail. If
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/cvsweb/back.gif){kind=icon}
Return to checksum-reading.diff CVS log ![[TXT]](/icons/cvsweb/text.gif){kind=icon}
![[DIR]](/icons/cvsweb/dir.gif){kind=icon}
Up to [ELWIX - Embedded LightWeight unIX -] / embedaddon / rsync / patches