1 /* Functions for dealing with sparse files
3 Copyright (C) 2003 Free Software Foundation, Inc.
5 This program is free software; you can redistribute it and/or modify it
6 under the terms of the GNU General Public License as published by the
7 Free Software Foundation; either version 2, or (at your option) any later
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
13 Public License for more details.
15 You should have received a copy of the GNU General Public License along
16 with this program; if not, write to the Free Software Foundation, Inc.,
17 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
23 struct tar_sparse_file
;
25 enum sparse_scan_state
32 struct tar_sparse_optab
34 bool (*init
) (struct tar_sparse_file
*);
35 bool (*done
) (struct tar_sparse_file
*);
36 bool (*sparse_member_p
) (struct tar_sparse_file
*);
37 bool (*dump_header
) (struct tar_sparse_file
*);
38 bool (*fixup_header
) (struct tar_sparse_file
*);
39 bool (*decode_header
) (struct tar_sparse_file
*);
40 bool (*scan_block
) (struct tar_sparse_file
*, enum sparse_scan_state
,
42 bool (*dump_region
) (struct tar_sparse_file
*, size_t index
);
43 bool (*extract_region
) (struct tar_sparse_file
*, size_t index
);
46 struct tar_sparse_file
48 int fd
; /* File descriptor */
49 size_t dumped_size
; /* Number of bytes actually written
51 struct tar_stat_info
*stat_info
; /* Information about the file */
52 struct tar_sparse_optab
*optab
;
53 void *closure
; /* Any additional data optab calls might
58 tar_sparse_member_p (struct tar_sparse_file
*file
)
60 if (file
->optab
->sparse_member_p
)
61 return file
->optab
->sparse_member_p (file
);
66 tar_sparse_init (struct tar_sparse_file
*file
)
68 file
->dumped_size
= 0;
69 if (file
->optab
->init
)
70 return file
->optab
->init (file
);
75 tar_sparse_done (struct tar_sparse_file
*file
)
77 if (file
->optab
->done
)
78 return file
->optab
->done (file
);
83 tar_sparse_scan (struct tar_sparse_file
*file
, enum sparse_scan_state state
,
86 if (file
->optab
->scan_block
)
87 return file
->optab
->scan_block (file
, state
, block
);
92 tar_sparse_dump_region (struct tar_sparse_file
*file
, size_t index
)
94 if (file
->optab
->dump_region
)
95 return file
->optab
->dump_region (file
, index
);
100 tar_sparse_extract_region (struct tar_sparse_file
*file
, size_t index
)
102 if (file
->optab
->extract_region
)
103 return file
->optab
->extract_region (file
, index
);
108 tar_sparse_dump_header (struct tar_sparse_file
*file
)
110 if (file
->optab
->dump_header
)
111 return file
->optab
->dump_header (file
);
116 tar_sparse_decode_header (struct tar_sparse_file
*file
)
118 if (file
->optab
->decode_header
)
119 return file
->optab
->decode_header (file
);
124 tar_sparse_fixup_header (struct tar_sparse_file
*file
)
126 if (file
->optab
->fixup_header
)
127 return file
->optab
->fixup_header (file
);
133 lseek_or_error (struct tar_sparse_file
*file
, off_t offset
, int whence
)
135 if (lseek (file
->fd
, offset
, whence
) < 0)
137 seek_diag_details (file
->stat_info
->orig_file_name
, offset
);
143 /* Takes a blockful of data and basically cruises through it to see if
144 it's made *entirely* of zeros, returning a 0 the instant it finds
145 something that is a nonzero, i.e., useful data. */
147 zero_block_p (char *buffer
, size_t size
)
155 #define clear_block(p) memset (p, 0, BLOCKSIZE);
157 #define SPARSES_INIT_COUNT SPARSES_IN_SPARSE_HEADER
160 sparse_add_map (struct tar_sparse_file
*file
, struct sp_array
*sp
)
162 if (file
->stat_info
->sparse_map
== NULL
)
164 file
->stat_info
->sparse_map
=
165 xmalloc (SPARSES_INIT_COUNT
* sizeof file
->stat_info
->sparse_map
[0]);
166 file
->stat_info
->sparse_map_size
= SPARSES_INIT_COUNT
;
168 else if (file
->stat_info
->sparse_map_avail
== file
->stat_info
->sparse_map_size
)
170 file
->stat_info
->sparse_map_size
*= 2;
171 file
->stat_info
->sparse_map
=
172 xrealloc (file
->stat_info
->sparse_map
,
173 file
->stat_info
->sparse_map_size
174 * sizeof file
->stat_info
->sparse_map
[0]);
176 file
->stat_info
->sparse_map
[file
->stat_info
->sparse_map_avail
++] = *sp
;
179 /* Scan the sparse file and create its map */
181 sparse_scan_file (struct tar_sparse_file
*file
)
183 static char buffer
[BLOCKSIZE
];
186 struct sp_array sp
= {0, 0};
188 if (!lseek_or_error (file
, 0, SEEK_SET
))
190 clear_block (buffer
);
192 file
->stat_info
->sparse_map_size
= 0;
193 file
->stat_info
->archive_file_size
= 0;
195 if (!tar_sparse_scan (file
, scan_begin
, NULL
))
198 while ((count
= safe_read (file
->fd
, buffer
, sizeof buffer
)) > 0)
200 /* Analize the block */
201 if (zero_block_p (buffer
, count
))
205 sparse_add_map (file
, &sp
);
207 if (!tar_sparse_scan (file
, scan_block
, NULL
))
213 if (sp
.numbytes
== 0)
215 sp
.numbytes
+= count
;
216 file
->stat_info
->archive_file_size
+= count
;
217 if (!tar_sparse_scan (file
, scan_block
, buffer
))
222 clear_block (buffer
);
225 if (sp
.numbytes
== 0)
228 sparse_add_map (file
, &sp
);
229 file
->stat_info
->archive_file_size
+= count
;
230 return tar_sparse_scan (file
, scan_end
, NULL
);
233 static struct tar_sparse_optab oldgnu_optab
;
234 static struct tar_sparse_optab star_optab
;
235 static struct tar_sparse_optab pax_optab
;
238 sparse_select_optab (struct tar_sparse_file
*file
)
240 switch (current_format
== DEFAULT_FORMAT
? archive_format
: current_format
)
247 case GNU_FORMAT
: /*FIXME: This one should disappear? */
248 file
->optab
= &oldgnu_optab
;
252 file
->optab
= &pax_optab
;
256 file
->optab
= &star_optab
;
266 sparse_dump_region (struct tar_sparse_file
*file
, size_t index
)
269 off_t bytes_left
= file
->stat_info
->sparse_map
[index
].numbytes
;
271 if (!lseek_or_error (file
, file
->stat_info
->sparse_map
[index
].offset
,
275 while (bytes_left
> 0)
277 size_t bufsize
= (bytes_left
> BLOCKSIZE
) ? BLOCKSIZE
: bytes_left
;
280 blk
= find_next_block ();
281 memset (blk
->buffer
, 0, BLOCKSIZE
);
282 bytes_read
= safe_read (file
->fd
, blk
->buffer
, bufsize
);
285 read_diag_details (file
->stat_info
->orig_file_name
,
286 file
->stat_info
->sparse_map
[index
].offset
287 + file
->stat_info
->sparse_map
[index
].numbytes
293 bytes_left
-= bytes_read
;
294 file
->dumped_size
+= bytes_read
;
295 set_next_block_after (blk
);
302 sparse_extract_region (struct tar_sparse_file
*file
, size_t index
)
306 if (!lseek_or_error (file
, file
->stat_info
->sparse_map
[index
].offset
,
310 write_size
= file
->stat_info
->sparse_map
[index
].numbytes
;
314 /* Last block of the file is a hole */
315 if (sys_truncate (file
->fd
))
316 truncate_warn (file
->stat_info
->orig_file_name
);
318 else while (write_size
> 0)
321 size_t wrbytes
= (write_size
> BLOCKSIZE
) ? BLOCKSIZE
: write_size
;
322 union block
*blk
= find_next_block ();
325 ERROR ((0, 0, _("Unexpected EOF in archive")));
328 set_next_block_after (blk
);
329 count
= full_write (file
->fd
, blk
->buffer
, wrbytes
);
331 file
->dumped_size
+= count
;
332 if (count
!= wrbytes
)
334 write_error_details (file
->stat_info
->orig_file_name
,
344 /* Interface functions */
346 sparse_dump_file (int fd
, struct tar_stat_info
*stat
)
349 struct tar_sparse_file file
;
351 file
.stat_info
= stat
;
354 if (!sparse_select_optab (&file
)
355 || !tar_sparse_init (&file
))
356 return dump_status_not_implemented
;
358 rc
= sparse_scan_file (&file
);
359 if (rc
&& file
.optab
->dump_region
)
361 tar_sparse_dump_header (&file
);
367 for (i
= 0; rc
&& i
< file
.stat_info
->sparse_map_avail
; i
++)
368 rc
= tar_sparse_dump_region (&file
, i
);
372 pad_archive(file
.stat_info
->archive_file_size
- file
.dumped_size
);
373 return (tar_sparse_done (&file
) && rc
) ? dump_status_ok
: dump_status_short
;
376 /* Returns true if the file represented by stat is a sparse one */
378 sparse_file_p (struct tar_stat_info
*stat
)
380 return (ST_NBLOCKS (stat
->stat
)
381 < (stat
->stat
.st_size
/ ST_NBLOCKSIZE
382 + (stat
->stat
.st_size
% ST_NBLOCKSIZE
!= 0)));
386 sparse_member_p (struct tar_stat_info
*stat
)
388 struct tar_sparse_file file
;
390 if (!sparse_select_optab (&file
))
392 file
.stat_info
= stat
;
393 return tar_sparse_member_p (&file
);
397 sparse_fixup_header (struct tar_stat_info
*stat
)
399 struct tar_sparse_file file
;
401 if (!sparse_select_optab (&file
))
403 file
.stat_info
= stat
;
404 return tar_sparse_fixup_header (&file
);
408 sparse_extract_file (int fd
, struct tar_stat_info
*stat
, off_t
*size
)
411 struct tar_sparse_file file
;
414 file
.stat_info
= stat
;
417 if (!sparse_select_optab (&file
)
418 || !tar_sparse_init (&file
))
419 return dump_status_not_implemented
;
421 rc
= tar_sparse_decode_header (&file
);
422 for (i
= 0; rc
&& i
< file
.stat_info
->sparse_map_avail
; i
++)
423 rc
= tar_sparse_extract_region (&file
, i
);
424 *size
= file
.stat_info
->archive_file_size
- file
.dumped_size
;
425 return (tar_sparse_done (&file
) && rc
) ? dump_status_ok
: dump_status_short
;
429 sparse_skip_file (struct tar_stat_info
*stat
)
432 struct tar_sparse_file file
;
434 file
.stat_info
= stat
;
437 if (!sparse_select_optab (&file
)
438 || !tar_sparse_init (&file
))
439 return dump_status_not_implemented
;
441 rc
= tar_sparse_decode_header (&file
);
442 skip_file (file
.stat_info
->archive_file_size
);
443 return (tar_sparse_done (&file
) && rc
) ? dump_status_ok
: dump_status_short
;
447 static char diff_buffer
[BLOCKSIZE
];
450 check_sparse_region (struct tar_sparse_file
*file
, off_t beg
, off_t end
)
452 if (!lseek_or_error (file
, beg
, SEEK_SET
))
458 size_t rdsize
= end
- beg
;
460 if (rdsize
> BLOCKSIZE
)
462 clear_block (diff_buffer
);
463 bytes_read
= safe_read (file
->fd
, diff_buffer
, rdsize
);
466 read_diag_details (file
->stat_info
->orig_file_name
,
471 if (!zero_block_p (diff_buffer
, bytes_read
))
473 report_difference (file
->stat_info
,
474 _("File fragment at %lu is not a hole"), beg
);
484 check_data_region (struct tar_sparse_file
*file
, size_t index
)
488 if (!lseek_or_error (file
, file
->stat_info
->sparse_map
[index
].offset
,
491 size_left
= file
->stat_info
->sparse_map
[index
].numbytes
;
492 while (size_left
> 0)
495 size_t rdsize
= (size_left
> BLOCKSIZE
) ? BLOCKSIZE
: size_left
;
497 union block
*blk
= find_next_block ();
500 ERROR ((0, 0, _("Unexpected EOF in archive")));
503 set_next_block_after (blk
);
504 bytes_read
= safe_read (file
->fd
, diff_buffer
, rdsize
);
507 read_diag_details (file
->stat_info
->orig_file_name
,
508 file
->stat_info
->sparse_map
[index
].offset
509 + file
->stat_info
->sparse_map
[index
].numbytes
514 file
->dumped_size
+= bytes_read
;
515 size_left
-= bytes_read
;
516 if (memcmp (blk
->buffer
, diff_buffer
, rdsize
))
518 report_difference (file
->stat_info
, _("Contents differ"));
526 sparse_diff_file (int fd
, struct tar_stat_info
*stat
)
529 struct tar_sparse_file file
;
533 file
.stat_info
= stat
;
536 if (!sparse_select_optab (&file
)
537 || !tar_sparse_init (&file
))
538 return dump_status_not_implemented
;
540 rc
= tar_sparse_decode_header (&file
);
541 for (i
= 0; rc
&& i
< file
.stat_info
->sparse_map_avail
; i
++)
543 rc
= check_sparse_region (&file
,
544 offset
, file
.stat_info
->sparse_map
[i
].offset
)
545 && check_data_region (&file
, i
);
546 offset
= file
.stat_info
->sparse_map
[i
].offset
547 + file
.stat_info
->sparse_map
[i
].numbytes
;
551 skip_file (file
.stat_info
->archive_file_size
- file
.dumped_size
);
553 tar_sparse_done (&file
);
558 /* Old GNU Format. The sparse file information is stored in the
559 oldgnu_header in the following manner:
561 The header is marked with type 'S'. Its `size' field contains
562 the cumulative size of all non-empty blocks of the file. The
563 actual file size is stored in `realsize' member of oldgnu_header.
565 The map of the file is stored in a list of `struct sparse'.
566 Each struct contains offset to the block of data and its
567 size (both as octal numbers). The first file header contains
568 at most 4 such structs (SPARSES_IN_OLDGNU_HEADER). If the map
569 contains more structs, then the field `isextended' of the main
570 header is set to 1 (binary) and the `struct sparse_header'
571 header follows, containing at most 21 following structs
572 (SPARSES_IN_SPARSE_HEADER). If more structs follow, `isextended'
573 field of the extended header is set and next next extension header
576 enum oldgnu_add_status
584 oldgnu_sparse_member_p (struct tar_sparse_file
*file_unused
)
586 return current_header
->header
.typeflag
== GNUTYPE_SPARSE
;
589 /* Add a sparse item to the sparse file and its obstack */
590 static enum oldgnu_add_status
591 oldgnu_add_sparse (struct tar_sparse_file
*file
, struct sparse
*s
)
595 if (s
->numbytes
[0] == '\0')
597 sp
.offset
= OFF_FROM_HEADER (s
->offset
);
598 sp
.numbytes
= SIZE_FROM_HEADER (s
->numbytes
);
600 || file
->stat_info
->stat
.st_size
< sp
.offset
+ sp
.numbytes
601 || file
->stat_info
->archive_file_size
< 0)
604 sparse_add_map (file
, &sp
);
609 oldgnu_fixup_header (struct tar_sparse_file
*file
)
611 /* NOTE! st_size was initialized from the header
612 which actually contains archived size. The following fixes it */
613 file
->stat_info
->archive_file_size
= file
->stat_info
->stat
.st_size
;
614 file
->stat_info
->stat
.st_size
=
615 OFF_FROM_HEADER (current_header
->oldgnu_header
.realsize
);
619 /* Convert old GNU format sparse data to internal representation */
621 oldgnu_get_sparse_info (struct tar_sparse_file
*file
)
624 union block
*h
= current_header
;
626 static enum oldgnu_add_status rc
;
628 file
->stat_info
->sparse_map_size
= 0;
629 for (i
= 0; i
< SPARSES_IN_OLDGNU_HEADER
; i
++)
631 rc
= oldgnu_add_sparse (file
, &h
->oldgnu_header
.sp
[i
]);
636 for (ext_p
= h
->oldgnu_header
.isextended
;
637 rc
== add_ok
&& ext_p
; ext_p
= h
->sparse_header
.isextended
)
639 h
= find_next_block ();
642 ERROR ((0, 0, _("Unexpected EOF in archive")));
645 set_next_block_after (h
);
646 for (i
= 0; i
< SPARSES_IN_SPARSE_HEADER
&& rc
== add_ok
; i
++)
647 rc
= oldgnu_add_sparse (file
, &h
->sparse_header
.sp
[i
]);
652 ERROR ((0, 0, _("%s: invalid sparse archive member"),
653 file
->stat_info
->orig_file_name
));
660 oldgnu_store_sparse_info (struct tar_sparse_file
*file
, size_t *pindex
,
661 struct sparse
*sp
, size_t sparse_size
)
663 for (; *pindex
< file
->stat_info
->sparse_map_avail
664 && sparse_size
> 0; sparse_size
--, sp
++, ++*pindex
)
666 OFF_TO_CHARS (file
->stat_info
->sparse_map
[*pindex
].offset
,
668 SIZE_TO_CHARS (file
->stat_info
->sparse_map
[*pindex
].numbytes
,
674 oldgnu_dump_header (struct tar_sparse_file
*file
)
676 off_t block_ordinal
= current_block_ordinal ();
680 blk
= start_header (file
->stat_info
);
681 blk
->header
.typeflag
= GNUTYPE_SPARSE
;
682 if (file
->stat_info
->sparse_map_avail
> SPARSES_IN_OLDGNU_HEADER
)
683 blk
->oldgnu_header
.isextended
= 1;
685 /* Store the real file size */
686 OFF_TO_CHARS (file
->stat_info
->stat
.st_size
, blk
->oldgnu_header
.realsize
);
687 /* Store the effective (shrunken) file size */
688 OFF_TO_CHARS (file
->stat_info
->archive_file_size
, blk
->header
.size
);
691 oldgnu_store_sparse_info (file
, &i
,
692 blk
->oldgnu_header
.sp
,
693 SPARSES_IN_OLDGNU_HEADER
);
694 blk
->oldgnu_header
.isextended
= i
< file
->stat_info
->sparse_map_avail
;
695 finish_header (file
->stat_info
, blk
, block_ordinal
);
697 while (i
< file
->stat_info
->sparse_map_avail
)
699 blk
= find_next_block ();
700 memset (blk
->buffer
, 0, BLOCKSIZE
);
701 oldgnu_store_sparse_info (file
, &i
,
702 blk
->sparse_header
.sp
,
703 SPARSES_IN_SPARSE_HEADER
);
704 set_next_block_after (blk
);
705 if (i
< file
->stat_info
->sparse_map_avail
)
706 blk
->sparse_header
.isextended
= 1;
713 static struct tar_sparse_optab oldgnu_optab
= {
714 NULL
, /* No init function */
715 NULL
, /* No done function */
716 oldgnu_sparse_member_p
,
719 oldgnu_get_sparse_info
,
720 NULL
, /* No scan_block function */
722 sparse_extract_region
,
729 star_sparse_member_p (struct tar_sparse_file
*file_unused
)
731 return current_header
->header
.typeflag
== GNUTYPE_SPARSE
;
735 star_fixup_header (struct tar_sparse_file
*file
)
737 /* NOTE! st_size was initialized from the header
738 which actually contains archived size. The following fixes it */
739 file
->stat_info
->archive_file_size
= file
->stat_info
->stat
.st_size
;
740 file
->stat_info
->stat
.st_size
=
741 OFF_FROM_HEADER (current_header
->star_in_header
.realsize
);
745 /* Convert STAR format sparse data to internal representation */
747 star_get_sparse_info (struct tar_sparse_file
*file
)
750 union block
*h
= current_header
;
752 static enum oldgnu_add_status rc
;
754 file
->stat_info
->sparse_map_size
= 0;
756 if (h
->star_in_header
.prefix
[0] == '\0'
757 && h
->star_in_header
.sp
[0].offset
[10] != '\0')
759 /* Old star format */
760 for (i
= 0; i
< SPARSES_IN_STAR_HEADER
; i
++)
762 rc
= oldgnu_add_sparse (file
, &h
->star_in_header
.sp
[i
]);
766 ext_p
= h
->star_in_header
.isextended
;
771 for (; rc
== add_ok
&& ext_p
; ext_p
= h
->star_ext_header
.isextended
)
773 h
= find_next_block ();
776 ERROR ((0, 0, _("Unexpected EOF in archive")));
779 set_next_block_after (h
);
780 for (i
= 0; i
< SPARSES_IN_STAR_EXT_HEADER
&& rc
== add_ok
; i
++)
781 rc
= oldgnu_add_sparse (file
, &h
->star_ext_header
.sp
[i
]);
786 ERROR ((0, 0, _("%s: invalid sparse archive member"),
787 file
->stat_info
->orig_file_name
));
794 static struct tar_sparse_optab star_optab
= {
795 NULL
, /* No init function */
796 NULL
, /* No done function */
797 star_sparse_member_p
,
800 star_get_sparse_info
,
801 NULL
, /* No scan_block function */
802 NULL
, /* No dump region function */
803 sparse_extract_region
,
807 /* GNU PAX sparse file format. The sparse file map is stored in
810 GNU.sparse.size Real size of the stored file
811 GNU.sparse.numblocks Number of blocks in the sparse map
812 repeat numblocks time
813 GNU.sparse.offset Offset of the next data block
814 GNU.sparse.numbytes Size of the next data block
819 pax_sparse_member_p (struct tar_sparse_file
*file
)
821 return file
->stat_info
->archive_file_size
!= file
->stat_info
->stat
.st_size
;
825 pax_dump_header (struct tar_sparse_file
*file
)
827 off_t block_ordinal
= current_block_ordinal ();
831 /* Store the real file size */
832 xheader_store ("GNU.sparse.size", file
->stat_info
, NULL
);
833 xheader_store ("GNU.sparse.numblocks", file
->stat_info
, NULL
);
834 for (i
= 0; i
< file
->stat_info
->sparse_map_avail
; i
++)
836 xheader_store ("GNU.sparse.offset", file
->stat_info
, &i
);
837 xheader_store ("GNU.sparse.numbytes", file
->stat_info
, &i
);
840 blk
= start_header (file
->stat_info
);
841 /* Store the effective (shrunken) file size */
842 OFF_TO_CHARS (file
->stat_info
->archive_file_size
, blk
->header
.size
);
843 finish_header (file
->stat_info
, blk
, block_ordinal
);
847 static struct tar_sparse_optab pax_optab
= {
848 NULL
, /* No init function */
849 NULL
, /* No done function */
852 NULL
, /* No decode_header function */
853 NULL
, /* No fixup_header function */
854 NULL
, /* No scan_block function */
856 sparse_extract_region
,