From 51aee274e892923a3f8fdb774e4f6b90bce47437 Mon Sep 17 00:00:00 2001 From: Sergey Poznyakoff Date: Sat, 8 Aug 2009 19:53:54 +0300 Subject: [PATCH] Fix handling of files removed during incremental dumps. Changes to src/create.c and src/incremen.c are partially based on patch from Alexander Peslyak . The new testcases require paxutils commit f653a2b or later. * src/common.h (struct name): New member `cmdline'. (dump_file): Change type of the 2nd argument to bool. (file_removed_diag, dir_removed_diag): New prototypes. (addname): New argument `cmdline'. (name_from_list): Change return value. * src/create.c (dump_dir0, dump_dir): top_level is bool. (create_archive): Update calls to name_from_list. Take advantage of the name->cmdline to set top_level argument during incremental backups. (dump_file0): top_level is bool. Do not bail out if a no-top-level file disappears during incremental backup, use file_removed_diag instead. (dump_filed): top_level is bool. * src/incremen.c (update_parent_directory): Silently ignore ENOENT. It should have already been reported elsewhere. (scan_directory): Use dir_removed_diag to report missing directories. * src/misc.c (file_removed_diag, dir_removed_diag): New functions. * src/names.c (name_gather): Set ->cmdname. (addname): Likewise. All uses updated. (name_from_list): Return struct name const *. All uses updated. * tests/filerem01.at: New testcase. * tests/filerem02.at: New testcase. * tests/Makefile.am, tests/testsuite.at: Add filerem01.at, filerem02.at * tests/grow.at, test/truncate.at: Use new syntax for genfile --run. * NEWS: Update. * doc/tar.texi: Minor fix. --- NEWS | 17 +++++++++ doc/tar.texi | 2 +- src/common.h | 17 ++++++--- src/create.c | 48 ++++++++++++------------- src/incremen.c | 14 ++++++-- src/misc.c | 24 +++++++++++++ src/names.c | 22 ++++++------ src/tar.c | 2 +- src/update.c | 6 ++-- tests/Makefile.am | 2 ++ tests/filerem01.at | 88 ++++++++++++++++++++++++++++++++++++++++++++++ tests/filerem02.at | 50 ++++++++++++++++++++++++++ tests/grow.at | 3 +- tests/testsuite.at | 4 +++ tests/truncate.at | 2 +- 15 files changed, 250 insertions(+), 51 deletions(-) create mode 100644 tests/filerem01.at create mode 100644 tests/filerem02.at diff --git a/NEWS b/NEWS index 352f5a2..93e2ac6 100644 --- a/NEWS +++ b/NEWS @@ -31,6 +31,23 @@ options. So far the only meaningful value for N is 0. The `--level=0' option forces creating the level 0 dump, by truncating the snapshot file if it exists. +* Files removed during incremental dumps + +If a file or directory is removed while incremental dump is +in progress, tar exact actions depend on whether this file +was explicitly listed in the command line, or was gathered +during file system scan. + +If the file was explicitly listed in the command line, tar +issues error messages and exits with the code 2, meaning +fatal error. + +Otherwise, if the file was gathered during the file system +scan, tar issues a warning, saying "File removed before we read it", +and sets the exit code to 1, which means "some files differ". +If the --warning=no-file-removed option is given, no warning +is issued and the exit code remains 0. + * Bugfixes ** Fix handling of hard link targets by -c --transform. ** Fix hard links recognition with -c --remove-files. diff --git a/doc/tar.texi b/doc/tar.texi index fdee5b5..7154dfd 100644 --- a/doc/tar.texi +++ b/doc/tar.texi @@ -2472,7 +2472,7 @@ messages as it reads through the archive. It is intended for when you want a visual indication that @command{tar} is still running, but don't want to see @option{--verbose} output. You can also instruct @command{tar} to execute a list of actions on each checkpoint, see -@option{--checklist-action} below. For a detailed description, see +@option{--checkpoint-action} below. For a detailed description, see @ref{checkpoints}. @opsummary{checkpoint-action} diff --git a/src/common.h b/src/common.h index 59de8a9..f2cdf2b 100644 --- a/src/common.h +++ b/src/common.h @@ -331,8 +331,10 @@ struct name char *name; /* File name or globbing pattern */ size_t length; /* cached strlen (name) */ - int matching_flags; /* wildcard flags if name is a pattern */ - + int matching_flags; /* wildcard flags if name is a pattern */ + bool cmdline; /* true if this name was given in the + command line */ + int change_dir; /* Number of the directory to change to. Set with the -C option. */ uintmax_t found_count; /* number of times a matching file has @@ -443,7 +445,7 @@ bool cachedir_file_p (const char *name); bool file_dumpable_p (struct tar_stat_info *st); void create_archive (void); void pad_archive (off_t size_left); -void dump_file (const char *st, int top_level, dev_t parent_device); +void dump_file (const char *st, bool top_level, dev_t parent_device); union block *start_header (struct tar_stat_info *st); void finish_header (struct tar_stat_info *st, union block *header, off_t block_ordinal); @@ -629,6 +631,10 @@ void readlink_diag (char const *name); void savedir_diag (char const *name); void seek_diag_details (char const *name, off_t offset); void stat_diag (char const *name); +void file_removed_diag (const char *name, bool top_level, + void (*diagfn) (char const *name)); +void dir_removed_diag (char const *name, bool top_level, + void (*diagfn) (char const *name)); void write_error_details (char const *name, size_t status, size_t size); void write_fatal (char const *name) __attribute__ ((noreturn)); void write_fatal_details (char const *name, ssize_t status, size_t size) @@ -656,12 +662,13 @@ void name_add_dir (const char *name); void name_term (void); const char *name_next (int change_dirs); void name_gather (void); -struct name *addname (char const *string, int change_dir, struct name *parent); +struct name *addname (char const *string, int change_dir, + bool cmdline, struct name *parent); bool name_match (const char *name); void names_notfound (void); void collect_and_sort_names (void); struct name *name_scan (const char *name); -char *name_from_list (void); +struct name const *name_from_list (void); void blank_name_list (void); char *new_name (const char *dir_name, const char *name); size_t stripped_prefix_len (char const *file_name, size_t num); diff --git a/src/create.c b/src/create.c index 3a0520c..e33122a 100644 --- a/src/create.c +++ b/src/create.c @@ -1092,7 +1092,7 @@ dump_regular_file (int fd, struct tar_stat_info *st) static void dump_dir0 (char *directory, - struct tar_stat_info *st, int top_level, dev_t parent_device) + struct tar_stat_info *st, bool top_level, dev_t parent_device) { dev_t our_device = st->stat.st_dev; const char *tag_file_name; @@ -1210,7 +1210,7 @@ dump_dir0 (char *directory, } strcpy (name_buf + name_len, entry); if (!excluded_name (name_buf)) - dump_file (name_buf, 0, our_device); + dump_file (name_buf, false, our_device); } free (name_buf); @@ -1224,7 +1224,7 @@ dump_dir0 (char *directory, name_buf = xmalloc (name_size); strcpy (name_buf, st->orig_file_name); strcat (name_buf, tag_file_name); - dump_file (name_buf, 0, our_device); + dump_file (name_buf, false, our_device); free (name_buf); break; @@ -1250,7 +1250,8 @@ ensure_slash (char **pstr) } static bool -dump_dir (int fd, struct tar_stat_info *st, int top_level, dev_t parent_device) +dump_dir (int fd, struct tar_stat_info *st, bool top_level, + dev_t parent_device) { char *directory = fdsavedir (fd); if (!directory) @@ -1271,7 +1272,7 @@ dump_dir (int fd, struct tar_stat_info *st, int top_level, dev_t parent_device) void create_archive (void) { - const char *p; + struct name const *p; open_archive (ACCESS_WRITE); buffer_write_global_xheader (); @@ -1285,21 +1286,21 @@ create_archive (void) collect_and_sort_names (); while ((p = name_from_list ()) != NULL) - if (!excluded_name (p)) - dump_file (p, -1, (dev_t) 0); + if (!excluded_name (p->name)) + dump_file (p->name, p->cmdline, (dev_t) 0); blank_name_list (); while ((p = name_from_list ()) != NULL) - if (!excluded_name (p)) + if (!excluded_name (p->name)) { - size_t plen = strlen (p); + size_t plen = strlen (p->name); if (buffer_size <= plen) { while ((buffer_size *= 2) <= plen) continue; buffer = xrealloc (buffer, buffer_size); } - memcpy (buffer, p, plen); + memcpy (buffer, p->name, plen); if (! ISSLASH (buffer[plen - 1])) buffer[plen++] = DIRECTORY_SEPARATOR; q = directory_contents (gnu_list_name->directory); @@ -1316,7 +1317,7 @@ create_archive (void) buffer = xrealloc (buffer, buffer_size); } strcpy (buffer + plen, q + 1); - dump_file (buffer, -1, (dev_t) 0); + dump_file (buffer, false, (dev_t) 0); } q += qlen + 1; } @@ -1325,9 +1326,10 @@ create_archive (void) } else { - while ((p = name_next (1)) != NULL) - if (!excluded_name (p)) - dump_file (p, 1, (dev_t) 0); + const char *name; + while ((name = name_next (1)) != NULL) + if (!excluded_name (name)) + dump_file (name, true, (dev_t) 0); } write_eot (); @@ -1475,7 +1477,6 @@ check_links (void) } } - /* Dump a single file, recursing on directories. P is the file name to dump. TOP_LEVEL tells whether this is a top-level call; zero means no, positive means yes, and negative means the top level @@ -1487,7 +1488,7 @@ check_links (void) static void dump_file0 (struct tar_stat_info *st, const char *p, - int top_level, dev_t parent_device) + bool top_level, dev_t parent_device) { union block *header; char type; @@ -1508,7 +1509,7 @@ dump_file0 (struct tar_stat_info *st, const char *p, if (deref_stat (dereference_option, p, &st->stat) != 0) { - stat_diag (p); + file_removed_diag (p, top_level, stat_diag); return; } st->archive_file_size = original_size = st->stat.st_size; @@ -1580,12 +1581,7 @@ dump_file0 (struct tar_stat_info *st, const char *p, : 0))); if (fd < 0) { - if (!top_level && errno == ENOENT) - WARNOPT (WARN_FILE_REMOVED, - (0, 0, _("%s: File removed before we read it"), - quotearg_colon (p))); - else - open_diag (p); + file_removed_diag (p, top_level, open_diag); return; } } @@ -1655,7 +1651,7 @@ dump_file0 (struct tar_stat_info *st, const char *p, : fstat (fd, &final_stat)) != 0) { - stat_diag (p); + file_removed_diag (p, top_level, stat_diag); ok = false; } } @@ -1713,7 +1709,7 @@ dump_file0 (struct tar_stat_info *st, const char *p, size = readlink (p, buffer, linklen + 1); if (size < 0) { - readlink_diag (p); + file_removed_diag (p, top_level, readlink_diag); return; } buffer[size] = '\0'; @@ -1795,7 +1791,7 @@ dump_file0 (struct tar_stat_info *st, const char *p, } void -dump_file (const char *p, int top_level, dev_t parent_device) +dump_file (const char *p, bool top_level, dev_t parent_device) { struct tar_stat_info st; tar_stat_init (&st); diff --git a/src/incremen.c b/src/incremen.c index aaeda58..02cae38 100644 --- a/src/incremen.c +++ b/src/incremen.c @@ -413,7 +413,11 @@ update_parent_directory (const char *name) { struct stat st; if (deref_stat (dereference_option, p, &st) != 0) - stat_diag (name); + { + if (errno != ENOENT) + stat_diag (directory->name); + /* else: should have been already reported */ + } else directory->mtime = get_stat_mtime (&st); } @@ -549,6 +553,12 @@ procdir (const char *name_buffer, struct stat *stat_data, if (one_file_system_option && device != stat_data->st_dev /* ... except if it was explicitely given in the command line */ && !is_individual_file (name_buffer)) + /* FIXME: + WARNOPT (WARN_XDEV, + (0, 0, + _("%s: directory is on a different filesystem; not dumped"), + quotearg_colon (directory->name))); + */ directory->children = NO_CHILDREN; else if (flag & PD_FORCE_CHILDREN) { @@ -699,7 +709,7 @@ scan_directory (char *dir, dev_t device, bool cmdline) if (deref_stat (dereference_option, name_buffer, &stat_data)) { - stat_diag (name_buffer); + dir_removed_diag (name_buffer, false, stat_diag); /* FIXME: used to be children = CHANGED_CHILDREN; but changed to: */ diff --git a/src/misc.c b/src/misc.c index a76b24e..b11b20e 100644 --- a/src/misc.c +++ b/src/misc.c @@ -745,6 +745,30 @@ stat_diag (char const *name) stat_error (name); } +void +file_removed_diag (const char *name, bool top_level, + void (*diagfn) (char const *name)) +{ + if (!top_level && errno == ENOENT) + WARNOPT (WARN_FILE_REMOVED, + (0, 0, _("%s: File removed before we read it"), + quotearg_colon (name))); + else + diagfn (name); +} + +void +dir_removed_diag (const char *name, bool top_level, + void (*diagfn) (char const *name)) +{ + if (!top_level && errno == ENOENT) + WARNOPT (WARN_FILE_REMOVED, + (0, 0, _("%s: Directory removed before we read it"), + quotearg_colon (name))); + else + diagfn (name); +} + void write_fatal_details (char const *name, ssize_t status, size_t size) { diff --git a/src/names.c b/src/names.c index 78cb543..16566bb 100644 --- a/src/names.c +++ b/src/names.c @@ -420,12 +420,13 @@ name_gather (void) buffer->matching_flags = matching_flags; buffer->directory = NULL; buffer->parent = NULL; + buffer->cmdline = true; namelist = buffer; nametail = &namelist->next; } else if (change_dir) - addname (0, change_dir, NULL); + addname (0, change_dir, false, NULL); } else { @@ -439,11 +440,11 @@ name_gather (void) change_dir = chdir_arg (xstrdup (ep->v.name)); if (ep) - addname (ep->v.name, change_dir, NULL); + addname (ep->v.name, change_dir, true, NULL); else { if (change_dir != change_dir0) - addname (0, change_dir, NULL); + addname (NULL, change_dir, false, NULL); break; } } @@ -452,7 +453,7 @@ name_gather (void) /* Add a name to the namelist. */ struct name * -addname (char const *string, int change_dir, struct name *parent) +addname (char const *string, int change_dir, bool cmdline, struct name *parent) { struct name *name = make_name (string); @@ -463,6 +464,7 @@ addname (char const *string, int change_dir, struct name *parent) name->change_dir = change_dir; name->directory = NULL; name->parent = parent; + name->cmdline = cmdline; *nametail = name; nametail = &name->next; @@ -811,7 +813,7 @@ add_hierarchy_to_namelist (struct name *name, dev_t device, bool cmdline) namebuf = xrealloc (namebuf, allocated_length + 1); } strcpy (namebuf + name_length, string + 1); - np = addname (namebuf, change_dir, name); + np = addname (namebuf, change_dir, false, name); if (!child_head) child_head = np; else @@ -886,7 +888,7 @@ collect_and_sort_names (void) name_gather (); if (!namelist) - addname (".", 0, NULL); + addname (".", 0, false, NULL); if (listed_incremental_option) { @@ -1030,8 +1032,8 @@ name_scan (const char *file_name) find and return all the non-found names in the namelist. */ struct name *gnu_list_name; -char * -name_from_list (void) +struct name const * +name_from_list () { if (!gnu_list_name) gnu_list_name = namelist; @@ -1042,9 +1044,9 @@ name_from_list (void) { gnu_list_name->found_count++; chdir_do (gnu_list_name->change_dir); - return gnu_list_name->name; + return gnu_list_name; } - return 0; + return NULL; } void diff --git a/src/tar.c b/src/tar.c index 8d23a4f..943de84 100644 --- a/src/tar.c +++ b/src/tar.c @@ -1388,7 +1388,7 @@ parse_opt (int key, char *arg, struct argp_state *state) case 'K': starting_file_option = true; - addname (arg, 0, NULL); + addname (arg, 0, true, NULL); break; case ONE_FILE_SYSTEM_OPTION: diff --git a/src/update.c b/src/update.c index fa16193..ade4283 100644 --- a/src/update.c +++ b/src/update.c @@ -189,10 +189,10 @@ update_archive (void) output_start = current_block->buffer; { - char *file_name; - - while ((file_name = name_from_list ()) != NULL) + struct name const *p; + while ((p = name_from_list ()) != NULL) { + char *file_name = p->name; if (excluded_name (file_name)) continue; if (interactive_option && !confirm ("add", file_name)) diff --git a/tests/Makefile.am b/tests/Makefile.am index 77b213c..0e22bdf 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -69,6 +69,8 @@ TESTSUITE_AT = \ extrac06.at\ extrac07.at\ extrac08.at\ + filerem01.at\ + filerem02.at\ gzip.at\ grow.at\ incremental.at\ diff --git a/tests/filerem01.at b/tests/filerem01.at new file mode 100644 index 0000000..a0832a5 --- /dev/null +++ b/tests/filerem01.at @@ -0,0 +1,88 @@ +# Process this file with autom4te to create testsuite. -*- Autotest -*- + +# Test suite for GNU tar. +# Copyright (C) 2009 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +AT_SETUP([file removed as we read it (ca. 22 seconds)]) +AT_KEYWORDS([create incremental filechange filerem filerem01]) + +AT_TAR_CHECK([ +mkdir dir +mkdir dir/sub +genfile --file dir/file1 +genfile --file dir/sub/file2 + +genfile --run --checkpoint=3 --unlink dir/file1 -- \ + tar --blocking-factor=1 --checkpoint=1 --checkpoint-action='sleep=1' \ + --checkpoint-action='echo' -c -f archive.tar \ + --listed-incremental db -v dir >/dev/null +], +[0], +[ignore], +[tar: dir: Directory is new +tar: dir/sub: Directory is new +tar: dir/file1: File removed before we read it +],[],[],[gnu, posix]) + +# Timing information: +# +# For -Hgnu the above command line takes about 8 seconds to execute and +# produces: +# +# tar: dir: Directory is new +# tar: dir/sub: Directory is new +# dir/ +# tar: Write checkpoint 1 +# tar: Write checkpoint 2 +# dir/sub/ +# tar: Write checkpoint 3 +# tar: Write checkpoint 4 +# dir/file1 +# tar: Write checkpoint 5 +# dir/sub/file2 +# tar: Write checkpoint 6 +# tar: Write checkpoint 7 +# tar: Write checkpoint 8 +# +# For -Hposix the above command line takes about 14 seconds to execute and +# produces: +# +# ./tar: dir: Directory is new +# ./tar: dir/sub: Directory is new +# dir/ +# ./tar: Write checkpoint 1 +# ./tar: Write checkpoint 2 +# ./tar: Write checkpoint 3 +# dir/sub/ +# ./tar: Write checkpoint 4 +# ./tar: Write checkpoint 5 +# ./tar: Write checkpoint 6 +# dir/file1 +# ./tar: Write checkpoint 7 +# ./tar: Write checkpoint 8 +# ./tar: Write checkpoint 9 +# dir/sub/file2 +# ./tar: Write checkpoint 10 +# ./tar: Write checkpoint 11 +# ./tar: Write checkpoint 12 +# ./tar: Write checkpoint 13 +# ./tar: Write checkpoint 14 + + +AT_CLEANUP + diff --git a/tests/filerem02.at b/tests/filerem02.at new file mode 100644 index 0000000..8d7005a --- /dev/null +++ b/tests/filerem02.at @@ -0,0 +1,50 @@ +# Process this file with autom4te to create testsuite. -*- Autotest -*- + +# Test suite for GNU tar. +# Copyright (C) 2009 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Description: see filerem01.at +# This test case checks if the tar exit code is still 2 if a +# file or directory disappears that is explicitly mentioned +# in the command line. + +AT_SETUP([toplevel file removed (ca. 24 seconds)]) +AT_KEYWORDS([create incremental filechange filerem filerem02]) + +AT_TAR_CHECK([ +mkdir dir +mkdir dir/sub +genfile --file dir/file1 +genfile --file dir/sub/file2 +mkdir dir2 +genfile --file dir2/file1 + +genfile --run --checkpoint=3 --exec 'rm -rf dir2' -- \ + tar --blocking-factor=1 --checkpoint=1 --checkpoint-action='sleep=1' \ + --checkpoint-action='echo' -c -f archive.tar \ + --listed-incremental db -v --warning=no-new-dir dir dir2 >/dev/null +], +[2], +[ignore], +[tar: dir2: Cannot stat: No such file or directory +tar: dir2/file1: File removed before we read it +tar: Exiting with failure status due to previous errors +],[],[],[gnu, posix]) + +# Timing information: see filerem01.at + +AT_CLEANUP + diff --git a/tests/grow.at b/tests/grow.at index 6f71cc9..27a5fba 100644 --- a/tests/grow.at +++ b/tests/grow.at @@ -27,8 +27,7 @@ AT_KEYWORDS([grow filechange]) AT_TAR_CHECK([ genfile --file foo --length 50000k genfile --file baz -genfile --run 'tar -vcf bar foo baz' --checkpoint 10 --length 1024 \ - --append foo +genfile --run --checkpoint 10 --length 1024 --append foo -- tar --checkpoint -vcf bar foo baz ], [1], [foo diff --git a/tests/testsuite.at b/tests/testsuite.at index 9634c28..b22850e 100644 --- a/tests/testsuite.at +++ b/tests/testsuite.at @@ -154,6 +154,10 @@ m4_include([incr03.at]) m4_include([incr04.at]) m4_include([incr05.at]) m4_include([incr06.at]) + +m4_include([filerem01.at]) +m4_include([filerem02.at]) + m4_include([rename01.at]) m4_include([rename02.at]) m4_include([rename03.at]) diff --git a/tests/truncate.at b/tests/truncate.at index 8ff2f34..65f1e34 100644 --- a/tests/truncate.at +++ b/tests/truncate.at @@ -32,7 +32,7 @@ AT_KEYWORDS([truncate filechange]) AT_TAR_CHECK([ genfile --file foo --length 50000k genfile --file baz -genfile --run 'tar -vcf bar foo baz' --checkpoint 10 --length 49995k --truncate foo +genfile --run --checkpoint 10 --length 49995k --truncate foo -- tar --checkpoint -vcf bar foo baz echo Exit status: $? echo separator sleep 1 -- 2.45.2