diff options
author | Ondrej Oprala <ooprala@redhat.com> | 2013-01-22 14:21:23 +0100 |
---|---|---|
committer | Pádraig Brady <P@draigBrady.com> | 2013-04-14 01:00:45 +0100 |
commit | 8901e010fa24ba375d53e3e37fc7819621adcf19 (patch) | |
tree | 1f20f49bb0c455525b042e838462d4396c97a314 | |
parent | 4114c93af398d7aecb5eb253f90d9b4cc0785643 (diff) | |
download | coreutils-8901e010fa24ba375d53e3e37fc7819621adcf19.tar.xz |
dd: avoid buffer allocations unless needed
* src/dd.c: Add new static global variable ibuf.
(alloc_ibuf, alloc_obuf): New functions factored from dd_copy().
(dd_copy): Call the new functions to allocate memory for
ibuf and obuf when necessary.
(skip): Likewise.
* tests/dd/no-allocate.sh: New test.
* tests/local.mk: Reference the test.
-rw-r--r-- | src/dd.c | 135 | ||||
-rwxr-xr-x | tests/dd/no-allocate.sh | 53 | ||||
-rw-r--r-- | tests/local.mk | 1 |
3 files changed, 136 insertions, 53 deletions
@@ -236,6 +236,9 @@ static uintmax_t r_truncate = 0; static char newline_character = '\n'; static char space_character = ' '; +/* Input buffer. */ +static char *ibuf; + /* Output buffer. */ static char *obuf; @@ -646,6 +649,65 @@ Options are:\n\ exit (status); } +static char * +human_size (size_t n) +{ + static char hbuf[LONGEST_HUMAN_READABLE + 1]; + int human_opts = + (human_autoscale | human_round_to_nearest | human_base_1024 + | human_space_before_unit | human_SI | human_B); + return human_readable (n, hbuf, human_opts, 1, 1); +} + +/* Ensure input buffer IBUF is allocated. */ + +static void +alloc_ibuf (void) +{ + if (ibuf) + return; + + char *real_buf = malloc (input_blocksize + INPUT_BLOCK_SLOP); + if (!real_buf) + error (EXIT_FAILURE, 0, + _("memory exhausted by input buffer of size %zu bytes (%s)"), + input_blocksize, human_size (input_blocksize)); + + real_buf += SWAB_ALIGN_OFFSET; /* allow space for swab */ + + ibuf = ptr_align (real_buf, page_size); +} + +/* Ensure output buffer OBUF is allocated/initialized. */ + +static void +alloc_obuf (void) +{ + if (obuf) + return; + + if (conversions_mask & C_TWOBUFS) + { + /* Page-align the output buffer, too. */ + char *real_obuf = malloc (output_blocksize + OUTPUT_BLOCK_SLOP); + if (!real_obuf) + error (EXIT_FAILURE, 0, + _("memory exhausted by output buffer of size %zu bytes (%s)"), + output_blocksize, human_size (output_blocksize)); + obuf = ptr_align (real_obuf, page_size); + } + else + { + alloc_ibuf (); + obuf = ibuf; + } + + /* Write a sentinel to the slop after the buffer, + to allow efficient checking for NUL blocks. */ + assert (sizeof (uintptr_t) <= OUTPUT_BLOCK_SLOP); + memset (obuf + output_blocksize, 1, sizeof (uintptr_t)); +} + static void translate_charset (char const *new_trans) { @@ -1526,7 +1588,7 @@ skip_via_lseek (char const *filename, int fdesc, off_t offset, int whence) /* Throw away RECORDS blocks of BLOCKSIZE bytes plus BYTES bytes on file descriptor FDESC, which is open with read permission for FILE. - Store up to BLOCKSIZE bytes of the data at a time in BUF, if + Store up to BLOCKSIZE bytes of the data at a time in IBUF or OBUF, if necessary. RECORDS or BYTES must be nonzero. If FDESC is STDIN_FILENO, advance the input offset. Return the number of records remaining, i.e., that were not skipped because EOF was @@ -1535,7 +1597,7 @@ skip_via_lseek (char const *filename, int fdesc, off_t offset, int whence) static uintmax_t skip (int fdesc, char const *file, uintmax_t records, size_t blocksize, - size_t *bytes, char *buf) + size_t *bytes) { uintmax_t offset = records * blocksize + *bytes; @@ -1607,6 +1669,18 @@ skip (int fdesc, char const *file, uintmax_t records, size_t blocksize, } /* else file_size && offset > OFF_T_MAX or file ! seekable */ + char *buf; + if (fdesc == STDIN_FILENO) + { + alloc_ibuf (); + buf = ibuf; + } + else + { + alloc_obuf (); + buf = obuf; + } + do { ssize_t nread = iread_fnc (fdesc, buf, records ? blocksize : *bytes); @@ -1823,26 +1897,12 @@ set_fd_flags (int fd, int add_flags, char const *name) } } -static char * -human_size (size_t n) -{ - static char hbuf[LONGEST_HUMAN_READABLE + 1]; - int human_opts = - (human_autoscale | human_round_to_nearest | human_base_1024 - | human_space_before_unit | human_SI | human_B); - return human_readable (n, hbuf, human_opts, 1, 1); -} - /* The main loop. */ static int dd_copy (void) { - char *ibuf, *bufstart; /* Input buffer. */ - /* These are declared static so that even though we don't free the - buffers, valgrind will recognize that there is no "real" leak. */ - static char *real_buf; /* real buffer address before alignment */ - static char *real_obuf; + char *bufstart; /* Input buffer. */ ssize_t nread; /* Bytes read in the current block. */ /* If nonzero, then the previously read block was partial and @@ -1869,45 +1929,12 @@ dd_copy (void) It is necessary when accessing raw (i.e. character special) disk devices on Unixware or other SVR4-derived system. */ - real_buf = malloc (input_blocksize + INPUT_BLOCK_SLOP); - if (!real_buf) - error (EXIT_FAILURE, 0, - _("memory exhausted by input buffer of size %zu bytes (%s)"), - input_blocksize, human_size (input_blocksize)); - - ibuf = real_buf; - ibuf += SWAB_ALIGN_OFFSET; /* allow space for swab */ - - ibuf = ptr_align (ibuf, page_size); - - if (conversions_mask & C_TWOBUFS) - { - /* Page-align the output buffer, too. */ - real_obuf = malloc (output_blocksize + OUTPUT_BLOCK_SLOP); - if (!real_obuf) - error (EXIT_FAILURE, 0, - _("memory exhausted by output buffer of size %zu bytes (%s)"), - output_blocksize, human_size (output_blocksize)); - obuf = ptr_align (real_obuf, page_size); - } - else - { - real_obuf = NULL; - obuf = ibuf; - } - - /* Write a sentinel to the slop after the buffer, - to allow efficient checking for NUL blocks. */ - assert (sizeof (uintptr_t) <= OUTPUT_BLOCK_SLOP); - memset (obuf + output_blocksize, 1, sizeof (uintptr_t)); - if (skip_records != 0 || skip_bytes != 0) { uintmax_t us_bytes = input_offset + (skip_records * input_blocksize) + skip_bytes; uintmax_t us_blocks = skip (STDIN_FILENO, input_file, - skip_records, input_blocksize, &skip_bytes, - ibuf); + skip_records, input_blocksize, &skip_bytes); us_bytes -= input_offset; /* POSIX doesn't say what to do when dd detects it has been @@ -1927,8 +1954,7 @@ dd_copy (void) { size_t bytes = seek_bytes; uintmax_t write_records = skip (STDOUT_FILENO, output_file, - seek_records, output_blocksize, &bytes, - obuf); + seek_records, output_blocksize, &bytes); if (write_records != 0 || bytes != 0) { @@ -1955,6 +1981,9 @@ dd_copy (void) if (max_records == 0 && max_bytes == 0) return exit_status; + alloc_ibuf (); + alloc_obuf (); + while (1) { if (r_partial + r_full >= max_records + !!max_bytes) diff --git a/tests/dd/no-allocate.sh b/tests/dd/no-allocate.sh new file mode 100755 index 000000000..e45dd5cf9 --- /dev/null +++ b/tests/dd/no-allocate.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# make sure that dd doesn't allocate memory unnecessarily + +# Copyright (C) 2013 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src +print_ver_ dd +require_ulimit_ + +# count and skip is zero, we don't need to allocate memory +(ulimit -v 20000; dd bs=30M count=0) || fail=1 +(ulimit -v 20000; dd ibs=30M count=0) || fail=1 +(ulimit -v 20000; dd obs=30M count=0) || fail=1 + + +# Use a fifo for which seek fails, but read does not +if mkfifo tape; then + # for non seekable output we need to allocate buffer when needed + echo 1 > tape& + (ulimit -v 20000; dd bs=30M skip=1 count=0 if=tape) && fail=1 + + echo 1 > tape& + (ulimit -v 20000; dd ibs=30M skip=1 count=0 if=tape) && fail=1 + + echo 1 > tape& + (ulimit -v 20000; dd obs=30M skip=1 count=0 if=tape) || fail=1 + + + # for non seekable output we need to allocate buffer when needed + echo 1 > tape& + (ulimit -v 20000; dd bs=30M seek=1 count=0 of=tape) && fail=1 + + echo 1 > tape& + (ulimit -v 20000; dd obs=30M seek=1 count=0 of=tape) && fail=1 + + echo 1 > tape& + (ulimit -v 20000; dd ibs=30M seek=1 count=0 of=tape) || fail=1 +fi + +Exit $fail diff --git a/tests/local.mk b/tests/local.mk index e3a72ab3a..0b019d951 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -465,6 +465,7 @@ all_tests = \ tests/df/skip-rootfs.sh \ tests/dd/direct.sh \ tests/dd/misc.sh \ + tests/dd/no-allocate.sh \ tests/dd/nocache.sh \ tests/dd/not-rewound.sh \ tests/dd/reblock.sh \ |