static void http_list_directory_include_file(buffer *out, int symlinks, buffer *path, const char *classname, int encode) {
int fd = fdevent_open_cloexec(path->ptr, symlinks, O_RDONLY, 0);
ssize_t rd;
char buf[8192];
if (-1 == fd) return;
if (encode) {
buffer_append_string_len(out, CONST_STR_LEN("<pre class=\""));
buffer_append_string(out, classname);
buffer_append_string_len(out, CONST_STR_LEN("\">"));
}
while ((rd = read(fd, buf, sizeof(buf))) > 0) {
if (encode) {
buffer_append_string_encoded(out, buf, (size_t)rd, ENCODING_MINIMAL_XML);
} else {
buffer_append_string_len(out, buf, (size_t)rd);
}
}
close(fd);
if (encode) {
buffer_append_string_len(out, CONST_STR_LEN("</pre>"));
}
}
static int li_rand_device_bytes (unsigned char *buf, int num)
{
/* randomness from these devices is cryptographically strong,
* unless /dev/urandom is low on entropy */
static const char * const devices[] = {
#ifdef __OpenBSD__
"/dev/arandom",
#endif
"/dev/urandom",
"/dev/random"
};
/* device files might not be available in chroot environment,
* so prefer syscall, if available */
if (0 == li_getentropy(buf, (size_t)num)) return 1;
for (unsigned int u = 0; u < sizeof(devices)/sizeof(devices[0]); ++u) {
/*(some systems might have symlink to another device; omit O_NOFOLLOW)*/
int fd = fdevent_open_cloexec(devices[u], O_RDONLY, 0);
if (fd >= 0) {
ssize_t rd = 0;
#ifdef RNDGETENTCNT
int entropy;
if (0 == ioctl(fd, (unsigned long)(RNDGETENTCNT), &entropy)
&& entropy >= num*8)
#endif
rd = read(fd, buf, (size_t)num);
close(fd);
if (rd == num) {
return 1;
}
}
}
return 0;
}
/* similar to network_write_file_chunk_mmap, but doesn't use send on windows (because we're on pipes),
* also mmaps and sends complete chunk instead of only small parts - the files
* are supposed to be temp files with reasonable chunk sizes.
*
* Also always use mmap; the files are "trusted", as we created them.
*/
static ssize_t cgi_write_file_chunk_mmap(server *srv, connection *con, int fd, chunkqueue *cq) {
chunk* const c = cq->first;
off_t offset, toSend, file_end;
ssize_t r;
size_t mmap_offset, mmap_avail;
char *data;
force_assert(NULL != c);
force_assert(FILE_CHUNK == c->type);
force_assert(c->offset >= 0 && c->offset <= c->file.length);
offset = c->file.start + c->offset;
toSend = c->file.length - c->offset;
file_end = c->file.start + c->file.length; /* offset to file end in this chunk */
if (0 == toSend) {
chunkqueue_remove_finished_chunks(cq);
return 0;
}
/*(simplified from network_write_no_mmap.c:network_open_file_chunk())*/
UNUSED(con);
if (-1 == c->file.fd) {
if (-1 == (c->file.fd = fdevent_open_cloexec(c->file.name->ptr, O_RDONLY, 0))) {
log_error_write(srv, __FILE__, __LINE__, "ssb", "open failed:", strerror(errno), c->file.name);
return -1;
}
}
/* (re)mmap the buffer if range is not covered completely */
if (MAP_FAILED == c->file.mmap.start
|| offset < c->file.mmap.offset
|| file_end > (off_t)(c->file.mmap.offset + c->file.mmap.length)) {
if (MAP_FAILED != c->file.mmap.start) {
munmap(c->file.mmap.start, c->file.mmap.length);
c->file.mmap.start = MAP_FAILED;
}
c->file.mmap.offset = mmap_align_offset(offset);
c->file.mmap.length = file_end - c->file.mmap.offset;
if (MAP_FAILED == (c->file.mmap.start = mmap(NULL, c->file.mmap.length, PROT_READ, MAP_PRIVATE, c->file.fd, c->file.mmap.offset))) {
if (toSend > 65536) toSend = 65536;
data = malloc(toSend);
force_assert(data);
if (-1 == lseek(c->file.fd, offset, SEEK_SET)
|| 0 >= (toSend = read(c->file.fd, data, toSend))) {
if (-1 == toSend) {
log_error_write(srv, __FILE__, __LINE__, "ssbdo", "lseek/read failed:",
strerror(errno), c->file.name, c->file.fd, offset);
} else { /*(0 == toSend)*/
log_error_write(srv, __FILE__, __LINE__, "sbdo", "unexpected EOF (input truncated?):",
c->file.name, c->file.fd, offset);
}
free(data);
return -1;
}
}
}
if (MAP_FAILED != c->file.mmap.start) {
force_assert(offset >= c->file.mmap.offset);
mmap_offset = offset - c->file.mmap.offset;
force_assert(c->file.mmap.length > mmap_offset);
mmap_avail = c->file.mmap.length - mmap_offset;
force_assert(toSend <= (off_t) mmap_avail);
data = c->file.mmap.start + mmap_offset;
}
r = write(fd, data, toSend);
if (MAP_FAILED == c->file.mmap.start) free(data);
if (r < 0) {
switch (errno) {
case EAGAIN:
case EINTR:
return 0;
case EPIPE:
case ECONNRESET:
return -2;
default:
log_error_write(srv, __FILE__, __LINE__, "ssd",
"write failed:", strerror(errno), fd);
return -1;
}
}
if (r >= 0) {
chunkqueue_mark_written(cq, r);
}
return r;
}
static int mod_deflate_file_chunk(server *srv, connection *con, handler_ctx *hctx, chunk *c, off_t st_size) {
off_t abs_offset;
off_t toSend = -1;
char *start;
#ifdef USE_MMAP
off_t we_want_to_mmap = 2 MByte;
off_t we_want_to_send = st_size;
volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */
#else
start = NULL;
#endif
if (-1 == c->file.fd) { /* open the file if not already open */
if (-1 == (c->file.fd = fdevent_open_cloexec(c->file.name->ptr, O_RDONLY, 0))) {
log_error_write(srv, __FILE__, __LINE__, "sbs", "open failed for:", c->file.name, strerror(errno));
return -1;
}
}
abs_offset = c->file.start + c->offset;
#ifdef USE_MMAP
/* mmap the buffer
* - first mmap
* - new mmap as the we are at the end of the last one */
if (c->file.mmap.start == MAP_FAILED ||
abs_offset == (off_t)(c->file.mmap.offset + c->file.mmap.length)) {
/* Optimizations for the future:
*
* adaptive mem-mapping
* the problem:
* we mmap() the whole file. If someone has alot large files and 32bit
* machine the virtual address area will be unrun and we will have a failing
* mmap() call.
* solution:
* only mmap 16M in one chunk and move the window as soon as we have finished
* the first 8M
*
* read-ahead buffering
* the problem:
* sending out several large files in parallel trashes the read-ahead of the
* kernel leading to long wait-for-seek times.
* solutions: (increasing complexity)
* 1. use madvise
* 2. use a internal read-ahead buffer in the chunk-structure
* 3. use non-blocking IO for file-transfers
* */
/* all mmap()ed areas are 512kb expect the last which might be smaller */
off_t to_mmap;
/* this is a remap, move the mmap-offset */
if (c->file.mmap.start != MAP_FAILED) {
munmap(c->file.mmap.start, c->file.mmap.length);
c->file.mmap.offset += we_want_to_mmap;
} else {
/* in case the range-offset is after the first mmap()ed area we skip the area */
c->file.mmap.offset = 0;
while (c->file.mmap.offset + we_want_to_mmap < c->file.start) {
c->file.mmap.offset += we_want_to_mmap;
}
}
/* length is rel, c->offset too, assume there is no limit at the mmap-boundaries */
to_mmap = (c->file.start + c->file.length) - c->file.mmap.offset;
if (to_mmap > we_want_to_mmap) to_mmap = we_want_to_mmap;
/* we have more to send than we can mmap() at once */
if (we_want_to_send > to_mmap) we_want_to_send = to_mmap;
if (MAP_FAILED == (c->file.mmap.start = mmap(0, (size_t)to_mmap, PROT_READ, MAP_SHARED, c->file.fd, c->file.mmap.offset))
&& (errno != EINVAL || MAP_FAILED == (c->file.mmap.start = mmap(0, (size_t)to_mmap, PROT_READ, MAP_PRIVATE, c->file.fd, c->file.mmap.offset)))) {
log_error_write(srv, __FILE__, __LINE__, "ssbd", "mmap failed:",
strerror(errno), c->file.name, c->file.fd);
return -1;
}
c->file.mmap.length = to_mmap;
#ifdef HAVE_MADVISE
/* don't advise files < 64Kb */
if (c->file.mmap.length > (64 KByte) &&
0 != madvise(c->file.mmap.start, c->file.mmap.length, MADV_WILLNEED)) {
log_error_write(srv, __FILE__, __LINE__, "ssbd", "madvise failed:",
strerror(errno), c->file.name, c->file.fd);
}
#endif
/* chunk_reset() or chunk_free() will cleanup for us */
}
/* to_send = abs_mmap_end - abs_offset */
toSend = (c->file.mmap.offset + c->file.mmap.length) - abs_offset;
if (toSend > we_want_to_send) toSend = we_want_to_send;
if (toSend < 0) {
log_error_write(srv, __FILE__, __LINE__, "soooo",
"toSend is negative:",
toSend,
c->file.mmap.length,
abs_offset,
c->file.mmap.offset);
force_assert(toSend < 0);
}
start = c->file.mmap.start;
mapped = 1;
#endif
if (MAP_FAILED == c->file.mmap.start) {
toSend = st_size;
if (toSend > 2 MByte) toSend = 2 MByte;
if (NULL == (start = malloc((size_t)toSend)) || -1 == lseek(c->file.fd, abs_offset, SEEK_SET) || toSend != read(c->file.fd, start, (size_t)toSend)) {
log_error_write(srv, __FILE__, __LINE__, "sbss", "reading", c->file.name, "failed:", strerror(errno));
free(start);
return -1;
}
abs_offset = 0;
}
#ifdef USE_MMAP
if (mapped) {
signal(SIGBUS, sigbus_handler);
sigbus_jmp_valid = 1;
if (0 != sigsetjmp(sigbus_jmp, 1)) {
sigbus_jmp_valid = 0;
log_error_write(srv, __FILE__, __LINE__, "sbd", "SIGBUS in mmap:",
c->file.name, c->file.fd);
return -1;
}
}
#endif
if (mod_deflate_compress(srv, con, hctx,
(unsigned char *)start + (abs_offset - c->file.mmap.offset), toSend) < 0) {
log_error_write(srv, __FILE__, __LINE__, "s",
"compress failed.");
toSend = -1;
}
#ifdef USE_MMAP
if (mapped)
sigbus_jmp_valid = 0;
else
#endif
free(start);
return toSend;
}
