--- embedaddon/rsync/compat.c 2016/11/01 09:54:32 1.1.1.3 +++ embedaddon/rsync/compat.c 2021/03/17 00:32:36 1.1.1.4 @@ -3,7 +3,7 @@ * * Copyright (C) Andrew Tridgell 1996 * Copyright (C) Paul Mackerras 1996 - * Copyright (C) 2004-2015 Wayne Davison + * Copyright (C) 2004-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,15 +20,8 @@ */ #include "rsync.h" +#include "itypes.h" -int remote_protocol = 0; -int file_extra_cnt = 0; /* count of file-list extras that everyone gets */ -int inc_recurse = 0; -int compat_flags = 0; -int use_safe_inc_flist = 0; -int want_xattr_optim = 0; -int proper_seed_order = 0; - extern int am_server; extern int am_sender; extern int local_server; @@ -40,32 +33,53 @@ extern int preallocate_files; extern int append_mode; extern int fuzzy_basis; extern int read_batch; +extern int write_batch; extern int delay_updates; extern int checksum_seed; extern int basis_dir_cnt; extern int prune_empty_dirs; extern int protocol_version; +extern int detect_renamed; extern int protect_args; extern int preserve_uid; extern int preserve_gid; +extern int preserve_atimes; +extern int preserve_crtimes; extern int preserve_acls; extern int preserve_xattrs; +extern int preserve_fileflags; +extern int xfer_flags_as_varint; extern int need_messages_from_generator; extern int delete_mode, delete_before, delete_during, delete_after; +extern int do_compression; +extern int do_compression_level; extern char *shell_cmd; extern char *partial_dir; -extern char *dest_option; extern char *files_from; extern char *filesfrom_host; +extern const char *checksum_choice; +extern const char *compress_choice; extern filter_rule_list filter_list; extern int need_unsorted_flist; #ifdef ICONV_OPTION extern iconv_t ic_send, ic_recv; extern char *iconv_opt; #endif +extern struct name_num_obj valid_checksums; +int remote_protocol = 0; +int file_extra_cnt = 0; /* count of file-list extras that everyone gets */ +int inc_recurse = 0; +int compat_flags = 0; +int use_safe_inc_flist = 0; +int want_xattr_optim = 0; +int proper_seed_order = 0; +int inplace_partial = 0; +int do_negotiated_strings = 0; +int xmit_id0_names = 0; + /* These index values are for the file-list's extra-attribute array. */ -int uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx; +int pathname_ndx, depth_ndx, atimes_ndx, crtimes_ndx, uid_ndx, gid_ndx, fileflags_ndx, acls_ndx, xattrs_ndx, unsort_ndx; int receiver_symlink_times = 0; /* receiver can set the time on a symlink */ int sender_symlink_iconv = 0; /* sender should convert symlink content */ @@ -74,12 +88,32 @@ int sender_symlink_iconv = 0; /* sender should convert int filesfrom_convert = 0; #endif +#define MAX_NSTR_STRLEN 256 + +struct name_num_obj valid_compressions = { + "compress", NULL, NULL, 0, 0, { +#ifdef SUPPORT_ZSTD + { CPRES_ZSTD, "zstd", NULL }, +#endif +#ifdef SUPPORT_LZ4 + { CPRES_LZ4, "lz4", NULL }, +#endif + { CPRES_ZLIBX, "zlibx", NULL }, + { CPRES_ZLIB, "zlib", NULL }, + { CPRES_NONE, "none", NULL }, + { 0, NULL, NULL } + } +}; + #define CF_INC_RECURSE (1<<0) #define CF_SYMLINK_TIMES (1<<1) #define CF_SYMLINK_ICONV (1<<2) #define CF_SAFE_FLIST (1<<3) #define CF_AVOID_XATTR_OPTIM (1<<4) #define CF_CHKSUM_SEED_FIX (1<<5) +#define CF_INPLACE_PARTIAL_DIR (1<<6) +#define CF_VARINT_FLIST_FLAGS (1<<7) +#define CF_ID0_NAMES (1<<8) static const char *client_info; @@ -127,6 +161,7 @@ void set_allow_inc_recurse(void) allow_inc_recurse = 0; else if (!am_sender && (delete_before || delete_after + || detect_renamed || delay_updates || prune_empty_dirs)) allow_inc_recurse = 0; else if (am_server && !local_server @@ -134,16 +169,408 @@ void set_allow_inc_recurse(void) allow_inc_recurse = 0; } +void parse_compress_choice(int final_call) +{ + if (valid_compressions.negotiated_name) + do_compression = valid_compressions.negotiated_num; + else if (compress_choice) { + struct name_num_item *nni = get_nni_by_name(&valid_compressions, compress_choice, -1); + if (!nni) { + rprintf(FERROR, "unknown compress name: %s\n", compress_choice); + exit_cleanup(RERR_UNSUPPORTED); + } + do_compression = nni->num; + if (am_server) + validate_choice_vs_env(NSTR_COMPRESS, do_compression, -1); + } else if (do_compression) + do_compression = CPRES_ZLIB; + else + do_compression = CPRES_NONE; + + if (do_compression != CPRES_NONE && final_call) + init_compression_level(); /* There's a chance this might turn compression off! */ + + if (do_compression == CPRES_NONE) + compress_choice = NULL; + + /* Snag the compression name for both write_batch's option output & the following debug output. */ + if (valid_compressions.negotiated_name) + compress_choice = valid_compressions.negotiated_name; + else if (compress_choice == NULL) { + struct name_num_item *nni = get_nni_by_num(&valid_compressions, do_compression); + compress_choice = nni ? nni->name : "UNKNOWN"; + } + + if (final_call && DEBUG_GTE(NSTR, am_server ? 3 : 1) + && (do_compression != CPRES_NONE || do_compression_level != CLVL_NOT_SPECIFIED)) { + rprintf(FINFO, "%s%s compress: %s (level %d)\n", + am_server ? "Server" : "Client", + valid_compressions.negotiated_name ? " negotiated" : "", + compress_choice, do_compression_level); + } +} + +struct name_num_item *get_nni_by_name(struct name_num_obj *nno, const char *name, int len) +{ + struct name_num_item *nni; + + if (len < 0) + len = strlen(name); + + for (nni = nno->list; nni->name; nni++) { + if (strncasecmp(name, nni->name, len) == 0 && nni->name[len] == '\0') + return nni; + } + + return NULL; +} + +struct name_num_item *get_nni_by_num(struct name_num_obj *nno, int num) +{ + struct name_num_item *nni; + + for (nni = nno->list; nni->name; nni++) { + if (num == nni->num) + return nni; + } + + return NULL; +} + +static void init_nno_saw(struct name_num_obj *nno, int val) +{ + struct name_num_item *nni; + int cnt; + + if (!nno->saw_len) { + for (nni = nno->list; nni->name; nni++) { + if (nni->num >= nno->saw_len) + nno->saw_len = nni->num + 1; + } + } + + if (!nno->saw) { + nno->saw = new_array0(uchar, nno->saw_len); + + /* We'll take this opportunity to make sure that the main_name values are set right. */ + for (cnt = 1, nni = nno->list; nni->name; nni++, cnt++) { + if (nno->saw[nni->num]) + nni->main_name = nno->list[nno->saw[nni->num]-1].name; + else + nno->saw[nni->num] = cnt; + } + } + + memset(nno->saw, val, nno->saw_len); +} + +/* Simplify the user-provided string so that it contains valid names without any duplicates. + * It also sets the "saw" flags to a 1-relative count of which name was seen first. */ +static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf, int tobuf_len) +{ + char *to = tobuf, *tok = NULL; + int saw_tok = 0, cnt = 0; + + while (1) { + int at_space = isSpace(from); + char ch = *from++; + if (ch == '&') + ch = '\0'; + if (!ch || at_space) { + if (tok) { + struct name_num_item *nni = get_nni_by_name(nno, tok, to - tok); + if (nni && !nno->saw[nni->num]) { + nno->saw[nni->num] = ++cnt; + if (nni->main_name) { + to = tok + strlcpy(tok, nni->main_name, tobuf_len - (tok - tobuf)); + if (to - tobuf >= tobuf_len) { + to = tok - 1; + break; + } + } + } else + to = tok - (tok != tobuf); + saw_tok = 1; + tok = NULL; + } + if (!ch) + break; + continue; + } + if (!tok) { + if (to != tobuf) + *to++ = ' '; + tok = to; + } + if (to - tobuf >= tobuf_len - 1) { + to = tok - (tok != tobuf); + break; + } + *to++ = ch; + } + *to = '\0'; + + if (saw_tok && to == tobuf) + return strlcpy(tobuf, "INVALID", MAX_NSTR_STRLEN); + + return to - tobuf; +} + +/* This routine is always called with a tmpbuf of MAX_NSTR_STRLEN length, but the + * buffer may be pre-populated with a "len" length string to use OR a len of -1 + * to tell us to read a string from the fd. */ +static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf, int len) +{ + struct name_num_item *ret = NULL; + + if (len < 0) + len = read_vstring(f_in, tmpbuf, MAX_NSTR_STRLEN); + + if (DEBUG_GTE(NSTR, am_server ? 3 : 2)) { + if (am_server) + rprintf(FINFO, "Client %s list (on server): %s\n", nno->type, tmpbuf); + else + rprintf(FINFO, "Server %s list (on client): %s\n", nno->type, tmpbuf); + } + + if (len > 0) { + struct name_num_item *nni; + int best = nno->saw_len; /* We want best == 1 from the client list, so start with a big number. */ + char *space, *tok = tmpbuf; + while (tok) { + while (*tok == ' ') tok++; /* Should be unneeded... */ + if (!*tok) + break; + if ((space = strchr(tok, ' ')) != NULL) + *space = '\0'; + nni = get_nni_by_name(nno, tok, -1); + if (space) { + *space = ' '; + tok = space + 1; + } else + tok = NULL; + if (!nni || !nno->saw[nni->num] || best <= nno->saw[nni->num]) + continue; + ret = nni; + best = nno->saw[nni->num]; + if (best == 1 || am_server) /* The server side stops at the first acceptable client choice */ + break; + } + if (ret) { + free(nno->saw); + nno->saw = NULL; + nno->negotiated_name = ret->main_name ? ret->main_name : ret->name; + nno->negotiated_num = ret->num; + return; + } + } + + if (!am_server || !do_negotiated_strings) { + char *cp = tmpbuf; + int j; + rprintf(FERROR, "Failed to negotiate a %s choice.\n", nno->type); + rprintf(FERROR, "%s list: %s\n", am_server ? "Client" : "Server", tmpbuf); + /* Recreate our original list from the saw values. This can't overflow our huge + * buffer because we don't have enough valid entries to get anywhere close. */ + for (j = 1, *cp = '\0'; j <= nno->saw_len; j++) { + struct name_num_item *nni; + for (nni = nno->list; nni->name; nni++) { + if (nno->saw[nni->num] == j) { + *cp++ = ' '; + cp += strlcpy(cp, nni->name, MAX_NSTR_STRLEN - (cp - tmpbuf)); + break; + } + } + } + if (!*tmpbuf) + strlcpy(cp, " INVALID", MAX_NSTR_STRLEN); + rprintf(FERROR, "%s list:%s\n", am_server ? "Server" : "Client", tmpbuf); + } + + exit_cleanup(RERR_UNSUPPORTED); +} + +static const char *getenv_nstr(int ntype) +{ + const char *env_str = getenv(ntype == NSTR_COMPRESS ? "RSYNC_COMPRESS_LIST" : "RSYNC_CHECKSUM_LIST"); + + /* When writing a batch file, we always negotiate an old-style choice. */ + if (write_batch) + env_str = ntype == NSTR_COMPRESS ? "zlib" : protocol_version >= 30 ? "md5" : "md4"; + + if (am_server && env_str) { + char *cp = strchr(env_str, '&'); + if (cp) + env_str = cp + 1; + } + + return env_str; +} + +void validate_choice_vs_env(int ntype, int num1, int num2) +{ + struct name_num_obj *nno = ntype == NSTR_COMPRESS ? &valid_compressions : &valid_checksums; + const char *list_str = getenv_nstr(ntype); + char tmpbuf[MAX_NSTR_STRLEN]; + + if (!list_str) + return; + + while (isSpace(list_str)) list_str++; + + if (!*list_str) + return; + + init_nno_saw(nno, 0); + parse_nni_str(nno, list_str, tmpbuf, MAX_NSTR_STRLEN); + + if (ntype == NSTR_CHECKSUM) /* If "md4" is in the env list, all the old MD4 choices are OK too. */ + nno->saw[CSUM_MD4_ARCHAIC] = nno->saw[CSUM_MD4_BUSTED] = nno->saw[CSUM_MD4_OLD] = nno->saw[CSUM_MD4]; + + if (!nno->saw[num1] || (num2 >= 0 && !nno->saw[num2])) { + rprintf(FERROR, "Your --%s-choice value (%s) was refused by the server.\n", + ntype == NSTR_COMPRESS ? "compress" : "checksum", + ntype == NSTR_COMPRESS ? compress_choice : checksum_choice); + exit_cleanup(RERR_UNSUPPORTED); + } + + free(nno->saw); + nno->saw = NULL; +} + +/* The saw buffer is initialized and used to store ordinal values from 1 to N + * for the order of the args in the array. If dup_markup == '\0', duplicates + * are removed otherwise the char is prefixed to the duplicate term and, if it + * is an opening paren/bracket/brace, the matching closing char is suffixed. + * "none" is removed on the client side unless dup_markup != '\0'. */ +int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, char dup_markup) +{ + struct name_num_item *nni; + int len = 0, cnt = 0; + char delim = '\0', post_delim; + + switch (dup_markup) { + case '(': post_delim = ')'; break; + case '[': post_delim = ']'; break; + case '{': post_delim = '}'; break; + default: post_delim = '\0'; break; + } + + init_nno_saw(nno, 0); + + for (nni = nno->list, len = 0; nni->name; nni++) { + if (nni->main_name) { + if (!dup_markup) + continue; + delim = dup_markup; + } + if (nni->num == 0 && !am_server && !dup_markup) + continue; + if (len) + to_buf[len++]= ' '; + if (delim) { + to_buf[len++]= delim; + delim = post_delim; + } + len += strlcpy(to_buf+len, nni->name, to_buf_len - len); + if (len >= to_buf_len - 3) + exit_cleanup(RERR_UNSUPPORTED); /* IMPOSSIBLE... */ + if (delim) { + to_buf[len++]= delim; + delim = '\0'; + } + nno->saw[nni->num] = ++cnt; + } + + return len; +} + +static void send_negotiate_str(int f_out, struct name_num_obj *nno, int ntype) +{ + char tmpbuf[MAX_NSTR_STRLEN]; + const char *list_str = getenv_nstr(ntype); + int len; + + if (list_str && *list_str) { + init_nno_saw(nno, 0); + len = parse_nni_str(nno, list_str, tmpbuf, MAX_NSTR_STRLEN); + list_str = tmpbuf; + } else + list_str = NULL; + + if (!list_str || !*list_str) + len = get_default_nno_list(nno, tmpbuf, MAX_NSTR_STRLEN, '\0'); + + if (DEBUG_GTE(NSTR, am_server ? 3 : 2)) { + if (am_server) + rprintf(FINFO, "Server %s list (on server): %s\n", nno->type, tmpbuf); + else + rprintf(FINFO, "Client %s list (on client): %s\n", nno->type, tmpbuf); + } + + /* Each side sends their list of valid names to the other side and then both sides + * pick the first name in the client's list that is also in the server's list. */ + if (do_negotiated_strings) + write_vstring(f_out, tmpbuf, len); +} + +static void negotiate_the_strings(int f_in, int f_out) +{ + /* We send all the negotiation strings before we start to read them to help avoid a slow startup. */ + + if (!checksum_choice) + send_negotiate_str(f_out, &valid_checksums, NSTR_CHECKSUM); + + if (do_compression && !compress_choice) + send_negotiate_str(f_out, &valid_compressions, NSTR_COMPRESS); + + if (valid_checksums.saw) { + char tmpbuf[MAX_NSTR_STRLEN]; + int len; + if (do_negotiated_strings) + len = -1; + else + len = strlcpy(tmpbuf, protocol_version >= 30 ? "md5" : "md4", MAX_NSTR_STRLEN); + recv_negotiate_str(f_in, &valid_checksums, tmpbuf, len); + } + + if (valid_compressions.saw) { + char tmpbuf[MAX_NSTR_STRLEN]; + int len; + if (do_negotiated_strings) + len = -1; + else + len = strlcpy(tmpbuf, "zlib", MAX_NSTR_STRLEN); + recv_negotiate_str(f_in, &valid_compressions, tmpbuf, len); + } + + /* If the other side is too old to negotiate, the above steps just made sure that + * the env didn't disallow the old algorithm. Mark things as non-negotiated. */ + if (!do_negotiated_strings) + valid_checksums.negotiated_name = valid_compressions.negotiated_name = NULL; +} + void setup_protocol(int f_out,int f_in) { - if (am_sender) - file_extra_cnt += PTR_EXTRA_CNT; + assert(file_extra_cnt == 0); + assert(EXTRA64_CNT == 2 || EXTRA64_CNT == 1); + + /* All int64 values must be set first so that they are guaranteed to be + * aligned for direct int64-pointer memory access. */ + if (preserve_atimes) + atimes_ndx = (file_extra_cnt += EXTRA64_CNT); + if (preserve_crtimes) + crtimes_ndx = (file_extra_cnt += EXTRA64_CNT); + if (am_sender) /* This is most likely in the in64 union as well. */ + pathname_ndx = (file_extra_cnt += PTR_EXTRA_CNT); else - file_extra_cnt++; + depth_ndx = ++file_extra_cnt; if (preserve_uid) uid_ndx = ++file_extra_cnt; if (preserve_gid) gid_ndx = ++file_extra_cnt; + if (preserve_fileflags || (force_change && !am_sender)) + fileflags_ndx = ++file_extra_cnt; if (preserve_acls && !am_sender) acls_ndx = ++file_extra_cnt; if (preserve_xattrs) @@ -207,16 +634,16 @@ void setup_protocol(int f_out,int f_in) append_mode = 2; if (preserve_acls && !local_server) { rprintf(FERROR, - "--acls requires protocol 30 or higher" - " (negotiated %d).\n", - protocol_version); + "--acls requires protocol 30 or higher" + " (negotiated %d).\n", + protocol_version); exit_cleanup(RERR_PROTOCOL); } if (preserve_xattrs && !local_server) { rprintf(FERROR, - "--xattrs requires protocol 30 or higher" - " (negotiated %d).\n", - protocol_version); + "--xattrs requires protocol 30 or higher" + " (negotiated %d).\n", + protocol_version); exit_cleanup(RERR_PROTOCOL); } } @@ -231,33 +658,33 @@ void setup_protocol(int f_out,int f_in) if (protocol_version < 29) { if (fuzzy_basis) { rprintf(FERROR, - "--fuzzy requires protocol 29 or higher" - " (negotiated %d).\n", - protocol_version); + "--fuzzy requires protocol 29 or higher" + " (negotiated %d).\n", + protocol_version); exit_cleanup(RERR_PROTOCOL); } if (basis_dir_cnt && inplace) { rprintf(FERROR, - "%s with --inplace requires protocol 29 or higher" - " (negotiated %d).\n", - dest_option, protocol_version); + "%s with --inplace requires protocol 29 or higher" + " (negotiated %d).\n", + alt_dest_opt(0), protocol_version); exit_cleanup(RERR_PROTOCOL); } if (basis_dir_cnt > 1) { rprintf(FERROR, - "Using more than one %s option requires protocol" - " 29 or higher (negotiated %d).\n", - dest_option, protocol_version); + "Using more than one %s option requires protocol" + " 29 or higher (negotiated %d).\n", + alt_dest_opt(0), protocol_version); exit_cleanup(RERR_PROTOCOL); } if (prune_empty_dirs) { rprintf(FERROR, - "--prune-empty-dirs requires protocol 29 or higher" - " (negotiated %d).\n", - protocol_version); + "--prune-empty-dirs requires protocol 29 or higher" + " (negotiated %d).\n", + protocol_version); exit_cleanup(RERR_PROTOCOL); } } else if (protocol_version >= 30) { @@ -275,13 +702,39 @@ void setup_protocol(int f_out,int f_in) compat_flags |= CF_AVOID_XATTR_OPTIM; if (local_server || strchr(client_info, 'C') != NULL) compat_flags |= CF_CHKSUM_SEED_FIX; - write_byte(f_out, compat_flags); - } else - compat_flags = read_byte(f_in); + if (local_server || strchr(client_info, 'I') != NULL) + compat_flags |= CF_INPLACE_PARTIAL_DIR; + if (local_server || strchr(client_info, 'u') != NULL) + compat_flags |= CF_ID0_NAMES; + if (local_server || strchr(client_info, 'v') != NULL) { + do_negotiated_strings = 1; + compat_flags |= CF_VARINT_FLIST_FLAGS; + } + if (strchr(client_info, 'V') != NULL) { /* Support a pre-release 'V' that got superseded */ + if (!write_batch) + compat_flags |= CF_VARINT_FLIST_FLAGS; + write_byte(f_out, compat_flags); + } else + write_varint(f_out, compat_flags); + } else { /* read_varint() is compatible with the older write_byte() when the 0x80 bit isn't on. */ + compat_flags = read_varint(f_in); + if (compat_flags & CF_VARINT_FLIST_FLAGS) + do_negotiated_strings = 1; + } /* The inc_recurse var MUST be set to 0 or 1. */ inc_recurse = compat_flags & CF_INC_RECURSE ? 1 : 0; want_xattr_optim = protocol_version >= 31 && !(compat_flags & CF_AVOID_XATTR_OPTIM); proper_seed_order = compat_flags & CF_CHKSUM_SEED_FIX ? 1 : 0; + xfer_flags_as_varint = compat_flags & CF_VARINT_FLIST_FLAGS ? 1 : 0; + xmit_id0_names = compat_flags & CF_ID0_NAMES ? 1 : 0; + if (!xfer_flags_as_varint && preserve_crtimes) { + fprintf(stderr, "Both rsync versions must be at least 3.2.0 for --crtimes.\n"); + exit_cleanup(RERR_PROTOCOL); + } + if (!xfer_flags_as_varint && preserve_fileflags) { + fprintf(stderr, "Both rsync versions must be at least 3.2.0 for --fileflags.\n"); + exit_cleanup(RERR_PROTOCOL); + } if (am_sender) { receiver_symlink_times = am_server ? strchr(client_info, 'L') != NULL @@ -299,18 +752,23 @@ void setup_protocol(int f_out,int f_in) if (inc_recurse && !allow_inc_recurse) { /* This should only be able to happen in a batch. */ fprintf(stderr, - "Incompatible options specified for inc-recursive %s.\n", - read_batch ? "batch file" : "connection"); + "Incompatible options specified for inc-recursive %s.\n", + read_batch ? "batch file" : "connection"); exit_cleanup(RERR_SYNTAX); } use_safe_inc_flist = (compat_flags & CF_SAFE_FLIST) || protocol_version >= 31; need_messages_from_generator = 1; + if (compat_flags & CF_INPLACE_PARTIAL_DIR) + inplace_partial = 1; #ifdef CAN_SET_SYMLINK_TIMES } else if (!am_sender) { receiver_symlink_times = 1; #endif } + if (read_batch) + do_negotiated_strings = 0; + if (need_unsorted_flist && (!am_sender || inc_recurse)) unsort_ndx = ++file_extra_cnt; @@ -331,6 +789,8 @@ void setup_protocol(int f_out,int f_in) } #endif + negotiate_the_strings(f_in, f_out); + if (am_server) { if (!checksum_seed) checksum_seed = time(NULL) ^ (getpid() << 6); @@ -338,4 +798,12 @@ void setup_protocol(int f_out,int f_in) } else { checksum_seed = read_int(f_in); } + + parse_checksum_choice(1); /* Sets checksum_type & xfersum_type */ + parse_compress_choice(1); /* Sets do_compression */ + + if (write_batch && !am_server) + write_batch_shell_file(); + + init_flist(); }