diff options
author | jb <jb@138bc75d-0d04-0410-961f-82ee72b054a4> | 2018-01-02 13:25:10 +0000 |
---|---|---|
committer | jb <jb@138bc75d-0d04-0410-961f-82ee72b054a4> | 2018-01-02 13:25:10 +0000 |
commit | 6d6d3070450e37bdd63534aa97d9d06d50c5d31a (patch) | |
tree | 6b7cdd6ee7fe044bcfc528125182dac1a43f868a /libgfortran | |
parent | 813a5ab539cf064b953553155949f5c2de40f9b9 (diff) |
PR libgfortran/83649 Chunk large reads and writes
It turns out that Linux never reads or writes more than 2147479552
bytes in a single syscall. For writes this is not a problem as
libgfortran already contains a loop around write() to handle short
writes. But for reads we cannot do this, since then read will hang if
we have a short read when reading from the terminal. Also, there are
reports that macOS fails I/O's larger than 2 GB. Thus, to work around
these issues do large reads/writes in chunks.
The testcase from the PR
program largewr
integer(kind=1) :: a(2_8**31+1)
a = 0
a(size(a, kind=8)) = 1
open(10, file="largewr.dat", access="stream", form="unformatted")
write (10) a
close(10)
a(size(a, kind=8)) = 2
open(10, file="largewr.dat", access="stream", form="unformatted")
read (10) a
if (a(size(a, kind=8)) == 1) then
print *, "All is well"
else
print *, "Oh no"
end if
end program largewr
fails on trunk but works with the patch.
Regtested on x86_64-pc-linux-gnu, committed to trunk.
libgfortran/ChangeLog:
2018-01-02 Janne Blomqvist <jb@gcc.gnu.org>
PR libgfortran/83649
* io/unix.c (MAX_CHUNK): New define.
(raw_read): For reads larger than MAX_CHUNK, loop.
(raw_write): Write no more than MAX_CHUNK bytes per iteration.
git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@256074 138bc75d-0d04-0410-961f-82ee72b054a4
Diffstat (limited to 'libgfortran')
-rw-r--r-- | libgfortran/ChangeLog | 7 | ||||
-rw-r--r-- | libgfortran/io/unix.c | 50 |
2 files changed, 49 insertions, 8 deletions
diff --git a/libgfortran/ChangeLog b/libgfortran/ChangeLog index 8a7b66c7351..f014f005557 100644 --- a/libgfortran/ChangeLog +++ b/libgfortran/ChangeLog @@ -1,3 +1,10 @@ +2018-01-02 Janne Blomqvist <jb@gcc.gnu.org> + + PR libgfortran/83649 + * io/unix.c (MAX_CHUNK): New define. + (raw_read): For reads larger than MAX_CHUNK, loop. + (raw_write): Write no more than MAX_CHUNK bytes per iteration. + 2017-12-29 Jerry DeLisle <jvdelisle@gcc.gnu.org> PR libgfortran/83613 diff --git a/libgfortran/io/unix.c b/libgfortran/io/unix.c index a07a3c9cea8..7a982b38554 100644 --- a/libgfortran/io/unix.c +++ b/libgfortran/io/unix.c @@ -292,18 +292,49 @@ raw_flush (unix_stream *s __attribute__ ((unused))) return 0; } +/* Write/read at most 2 GB - 4k chunks at a time. Linux never reads or + writes more than this, and there are reports that macOS fails for + larger than 2 GB as well. */ +#define MAX_CHUNK 2147479552 + static ssize_t raw_read (unix_stream *s, void *buf, ssize_t nbyte) { /* For read we can't do I/O in a loop like raw_write does, because that will break applications that wait for interactive I/O. We - still can loop around EINTR, though. */ - while (true) + still can loop around EINTR, though. This however causes a + problem for large reads which must be chunked, see comment above. + So assume that if the size is larger than the chunk size, we're + reading from a file and not the terminal. */ + if (nbyte <= MAX_CHUNK) { - ssize_t trans = read (s->fd, buf, nbyte); - if (trans == -1 && errno == EINTR) - continue; - return trans; + while (true) + { + ssize_t trans = read (s->fd, buf, nbyte); + if (trans == -1 && errno == EINTR) + continue; + return trans; + } + } + else + { + ssize_t bytes_left = nbyte; + char *buf_st = buf; + while (bytes_left > 0) + { + ssize_t to_read = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK; + ssize_t trans = read (s->fd, buf_st, to_read); + if (trans == -1) + { + if (errno == EINTR) + continue; + else + return trans; + } + buf_st += trans; + bytes_left -= trans; + } + return nbyte - bytes_left; } } @@ -317,10 +348,13 @@ raw_write (unix_stream *s, const void *buf, ssize_t nbyte) buf_st = (char *) buf; /* We must write in a loop since some systems don't restart system - calls in case of a signal. */ + calls in case of a signal. Also some systems might fail outright + if we try to write more than 2 GB in a single syscall, so chunk + up large writes. */ while (bytes_left > 0) { - trans = write (s->fd, buf_st, bytes_left); + ssize_t to_write = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK; + trans = write (s->fd, buf_st, to_write); if (trans == -1) { if (errno == EINTR) |