Yes, it's time once again to return to the subject of moving files.
With the recent changes to the communications code between the receiver
and the generator, there is now a non-clogging channel that we can use
to signal the sender when a file has been successfully transferred,
which allows us delete the original for all transferred files. I have
in the past waffled on whether this feature needs to be in rsync. I'm
currently back on the side that it's a nice thing to support. YMMV.
Here's a new implementation of the feature that adds a generic message
(MSG_SUCCESS) for the receiver to send back to the sender (through the
generator). I made the generator simply forward this message on to the
sender, but to do that it means that the generator must be in multi-
plexed IO mode, which it used to be only when it was on the server side.
My patch adds a new internal flag that lets the code request that the
generator have messaging capability even when it is on the client side
(the non-delete-specific variable makes the code more generic). The one
not-so-nice thing about this setup is that the sender process gets the
MSG_SUCCESS for a particular index when it is down in the I/O code, and
that code doesn't know about the file list. I decided to make this code
call a new function, successful_send(), which is back in sender.c. This
function is the one that handles translating the index into a file name
and deleting the source file (assuming that the delete_sent_files flag
is on, which is currently the only time that MSG_SUCCESS gets sent). I
also added a run-time flag to mark the items we sent off to the
receiver, just to make sure that nothing funny is going on in the
sequence of events (aside: the sender side has no copy-on-write issues
to make us avoid tweaking the flags).
So, feel free to take a look and see if you like what I've done.
..wayne..
-------------- next part --------------
Index: flist.c
--- flist.c 17 Jan 2004 01:16:49 -0000 1.165
+++ flist.c 17 Jan 2004 05:04:54 -0000
@@ -602,7 +602,7 @@ void receive_file_entry(struct file_stru
if (!file->basename)
out_of_memory("receive_file_entry 1");
- file->flags = flags;
+ file->flags = flags & LIVE_FLAGS;
file->length = read_longint(f);
if (!(flags & SAME_TIME))
modtime = (time_t)read_int(f);
Index: io.c
--- io.c 16 Jan 2004 16:31:47 -0000 1.119
+++ io.c 17 Jan 2004 05:04:54 -0000
@@ -222,6 +222,14 @@ static void read_msg_fd(void)
read_loop(fd, buf, 4);
redo_list_add(IVAL(buf,0));
break;
+ case MSG_SUCCESS:
+ if (len != 4) {
+ rprintf(FERROR, "invalid message %d:%d\n", tag, len);
+ exit_cleanup(RERR_STREAMIO);
+ }
+ read_loop(fd, buf, 4);
+ io_multiplex_write(MSG_SUCCESS, buf, 4);
+ break;
case MSG_INFO:
case MSG_ERROR:
case MSG_LOG:
@@ -637,6 +645,16 @@ static int read_unbuffered(int fd, char
}
read_loop(fd, buffer, remaining);
bufferIdx = 0;
+ break;
+ case MSG_SUCCESS:
+ if (remaining != 4) {
+ rprintf(FERROR, "invalid multi-message %d:%ld\n",
+ tag, (long)remaining);
+ exit_cleanup(RERR_STREAMIO);
+ }
+ read_loop(fd, line, 4);
+ successful_send(IVAL(line, 0));
+ remaining = 0;
break;
case MSG_INFO:
case MSG_ERROR:
Index: main.c
--- main.c 17 Jan 2004 05:04:04 -0000 1.181
+++ main.c 17 Jan 2004 05:04:54 -0000
@@ -41,6 +41,7 @@ extern int list_only;
extern int local_server;
extern int log_got_error;
extern int module_id;
+extern int need_messages_from_generator;
extern int orig_umask;
extern int preserve_hard_links;
extern int protocol_version;
@@ -558,6 +559,8 @@ void start_server(int f_in, int f_out, i
io_start_multiplex_out(f_out);
if (am_sender) {
+ if (need_messages_from_generator)
+ io_start_multiplex_in(f_in);
if (!read_batch) {
recv_exclude_list(f_in);
if (cvs_exclude)
@@ -623,6 +626,9 @@ int client_run(int f_in, int f_out, pid_
io_flush(FULL_FLUSH);
exit_cleanup(status);
}
+
+ if (need_messages_from_generator)
+ io_start_multiplex_out(f_out);
if (argc == 0) {
list_only = 1;
Index: options.c
--- options.c 15 Jan 2004 17:43:34 -0000 1.124
+++ options.c 17 Jan 2004 05:04:55 -0000
@@ -81,12 +81,14 @@ int copy_unsafe_links=0;
int size_only=0;
int bwlimit=0;
int delete_after=0;
+int delete_sent_files = 0;
int only_existing=0;
int opt_ignore_existing=0;
int max_delete=0;
int ignore_errors=0;
int modify_window=0;
int blocking_io=-1;
+int need_messages_from_generator = 0;
unsigned int block_size = 0;
@@ -245,6 +247,7 @@ void usage(enum logcode F)
rprintf(F," --delete delete files that don't
exist on the sending side\n");
rprintf(F," --delete-excluded also delete excluded files on
the receiving side\n");
rprintf(F," --delete-after receiver deletes after
transferring, not before\n");
+ rprintf(F," --delete-sent-files updated/sent files are removed
from sending side\n");
rprintf(F," --ignore-errors delete even if there are IO
errors\n");
rprintf(F," --max-delete=NUM don't delete more than NUM
files\n");
rprintf(F," --partial keep partially transferred
files\n");
@@ -294,8 +297,8 @@ void usage(enum logcode F)
}
enum {OPT_VERSION = 1000, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
- OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_LINK_DEST,
- OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW,
+ OPT_DELETE_AFTER, OPT_DELETE_EXCLUDED, OPT_DELETE_SENT_FILES,
+ OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_LINK_DEST, OPT_MODIFY_WINDOW,
OPT_READ_BATCH, OPT_WRITE_BATCH};
static struct poptOption long_options[] = {
@@ -313,6 +316,7 @@ static struct poptOption long_options[]
{"ignore-existing", 0, POPT_ARG_NONE, &opt_ignore_existing,
0, 0, 0 },
{"delete-after", 0, POPT_ARG_NONE, 0,
OPT_DELETE_AFTER, 0, 0 },
{"delete-excluded", 0, POPT_ARG_NONE, 0,
OPT_DELETE_EXCLUDED, 0, 0 },
+ {"delete-sent-files",0, POPT_ARG_NONE, 0,
OPT_DELETE_SENT_FILES, 0, 0 },
{"force", 0, POPT_ARG_NONE, &force_delete, 0, 0,
0 },
{"numeric-ids", 0, POPT_ARG_NONE, &numeric_ids, 0, 0, 0
},
{"exclude", 0, POPT_ARG_STRING, 0,
OPT_EXCLUDE, 0, 0 },
@@ -498,6 +502,11 @@ int parse_arguments(int *argc, const cha
delete_mode = 1;
break;
+ case OPT_DELETE_SENT_FILES:
+ delete_sent_files = 1;
+ need_messages_from_generator = 1;
+ break;
+
case OPT_EXCLUDE:
add_exclude(&exclude_list, poptGetOptArg(pc),
ADD_EXCLUDE);
@@ -902,6 +911,9 @@ void server_options(char **args,int *arg
args[ac++] = "--from0";
}
}
+
+ if (delete_sent_files)
+ args[ac++] = "--delete-sent-files";
*argc = ac;
}
Index: proto.h
--- proto.h 15 Jan 2004 07:42:23 -0000 1.172
+++ proto.h 17 Jan 2004 05:04:55 -0000
@@ -193,6 +193,7 @@ int set_perms(char *fname,struct file_st
void sig_int(void);
void finish_transfer(char *fname, char *fnametmp, struct file_struct *file);
void read_sum_head(int f, struct sum_struct *sum);
+void successful_send(int i);
void send_files(struct file_list *flist, int f_out, int f_in);
int try_bind_local(int s, int ai_family, int ai_socktype,
const char *bind_address);
Index: receiver.c
--- receiver.c 15 Jan 2004 07:42:25 -0000 1.63
+++ receiver.c 17 Jan 2004 05:04:55 -0000
@@ -39,6 +39,7 @@ extern char *backup_dir;
extern char *backup_suffix;
extern int backup_suffix_len;
extern int cleanup_got_literal;
+extern int delete_sent_files;
static void delete_one(char *fn, int is_dir)
{
@@ -287,7 +288,7 @@ int recv_files(int f_in,struct file_list
char *fname, fbuf[MAXPATHLEN];
char template[MAXPATHLEN];
char fnametmp[MAXPATHLEN];
- char *fnamecmp;
+ char *fnamecmp, numbuf[4];
char fnamecmpbuf[MAXPATHLEN];
struct map_struct *mapbuf;
int i;
@@ -461,16 +462,20 @@ int recv_files(int f_in,struct file_list
cleanup_disable();
- if (!recv_ok) {
+ if (recv_ok) {
+ if (delete_sent_files) {
+ SIVAL(numbuf, 0, i);
+ send_msg(MSG_SUCCESS, numbuf, 4);
+ }
+ } else {
if (csum_length == SUM_LENGTH) {
rprintf(FERROR,"ERROR: file corruption in %s. File changed during
transfer?\n",
full_fname(fname));
} else {
- char buf[4];
if (verbose > 1)
rprintf(FINFO,"redoing %s(%d)\n",fname,i);
- SIVAL(buf, 0, i);
- send_msg(MSG_REDO, buf, 4);
+ SIVAL(numbuf, 0, i);
+ send_msg(MSG_REDO, numbuf, 4);
}
}
}
Index: rsync.1
--- rsync.1 15 Jan 2004 17:45:53 -0000 1.155
+++ rsync.1 17 Jan 2004 05:04:56 -0000
@@ -349,6 +349,7 @@ to the detailed description below for a
--delete delete files that don\&'t exist on sender
--delete-excluded also delete excluded files on receiver
--delete-after receiver deletes after transfer, not before
+ --delete-sent-files updated/sent files are removed from sender
--ignore-errors delete even if there are IO errors
--max-delete=NUM don\&'t delete more than NUM files
--partial keep partially transferred files
@@ -672,6 +673,12 @@ By default rsync does file deletions on
receiving side before transferring files to try to ensure that there is
sufficient space on the receiving filesystem\&. If you want to delete
after transferring, use the --delete-after switch\&. Implies
--delete\&.
+.IP
+.IP "\fB--delete-sent-files\fP"
+This tells rsync to remove the source files
+on the sending side that are successfully transferred to the receiving
+side\&. Directories are not removed, nor are files that are identical on
+both systems\&.
.IP
.IP "\fB--ignore-errors\fP"
Tells --delete to go ahead and delete files
Index: rsync.h
--- rsync.h 15 Jan 2004 07:42:27 -0000 1.173
+++ rsync.h 17 Jan 2004 05:04:56 -0000
@@ -39,6 +39,7 @@
incompatible with older versions :-( */
#define CHAR_OFFSET 0
+/* These flags are used during the flist transfer. */
#define FLAG_DELETE (1<<0)
#define SAME_MODE (1<<1)
@@ -54,9 +55,14 @@
#define HAS_INODE_DATA (1<<9)
#define SAME_DEV (1<<10)
-/* What flags are relevant after the transfer of the flist is complete? */
+/* What flags above are relevant after the transfer of the flist? */
#define LIVE_FLAGS FLAG_DELETE
+/* These flist flags can be set after the flist is transferred. */
+
+/*#define FLAG_DELETE (1<<0) -- from the above list */
+#define FLAG_SENT (1<<1)
+
/* update this if you make incompatible changes */
#define PROTOCOL_VERSION 28
@@ -120,6 +126,7 @@ enum msgcode {
MSG_ERROR=FERROR, MSG_INFO=FINFO, MSG_LOG=FLOG, /* remote logging */
MSG_REDO=4, /* reprocess indicated flist index */
MSG_DONE=5, /* current phase is done */
+ MSG_SUCCESS=6, /* successfully updated indicated flist index */
};
#include "errcode.h"
Index: rsync.yo
--- rsync.yo 15 Jan 2004 17:45:53 -0000 1.139
+++ rsync.yo 17 Jan 2004 05:04:57 -0000
@@ -312,6 +312,7 @@ verb(
--delete delete files that don't exist on sender
--delete-excluded also delete excluded files on receiver
--delete-after receiver deletes after transfer, not before
+ --delete-sent-files updated/sent files are removed from sender
--ignore-errors delete even if there are IO errors
--max-delete=NUM don't delete more than NUM files
--partial keep partially transferred files
@@ -584,6 +585,11 @@ dit(bf(--delete-after)) By default rsync
receiving side before transferring files to try to ensure that there is
sufficient space on the receiving filesystem. If you want to delete
after transferring, use the --delete-after switch. Implies --delete.
+
+dit(bf(--delete-sent-files)) This tells rsync to remove the source files
+on the sending side that are successfully transferred to the receiving
+side. Directories are not removed, nor are files that are identical on
+both systems.
dit(bf(--ignore-errors)) Tells --delete to go ahead and delete files
even when there are IO errors.
Index: sender.c
--- sender.c 15 Jan 2004 08:56:33 -0000 1.34
+++ sender.c 17 Jan 2004 05:04:57 -0000
@@ -27,6 +27,7 @@ extern int dry_run;
extern int am_server;
extern int am_daemon;
extern int protocol_version;
+extern int delete_sent_files;
/**
@@ -104,7 +105,28 @@ static struct sum_struct *receive_sums(i
return s;
}
+static struct file_list *the_flist;
+void successful_send(int i)
+{
+ char fname[MAXPATHLEN];
+ struct file_struct *file;
+ int offset = 0;
+
+ if (!the_flist)
+ return;
+
+ file = the_flist->files[i];
+ if (!(file->flags & FLAG_SENT))
+ return; /* We didn't send it -- impossible! */
+ if (file->basedir) {
+ /* We know the name fits because we already sent it. */
+ offset = snprintf(fname, MAXPATHLEN, "%s/", file->basedir);
+ }
+ f_name_to(file, fname + offset, MAXPATHLEN - offset);
+ if (delete_sent_files && do_unlink(fname) == 0 && verbose >
0)
+ rprintf(FINFO, "sender removed %s\n", fname + offset);
+}
void send_files(struct file_list *flist, int f_out, int f_in)
{
@@ -129,6 +151,8 @@ void send_files(struct file_list *flist,
if (verbose > 2)
rprintf(FINFO, "send_files starting\n");
+ the_flist = flist;
+
while (1) {
int offset = 0;
@@ -307,6 +331,9 @@ void send_files(struct file_list *flist,
if (verbose > 2)
rprintf(FINFO, "sender finished %s\n", fname);
+
+ /* Flag that we actually sent this entry. */
+ file->flags |= FLAG_SENT;
}
if (verbose > 2)