static int reread_data(Chunk *chunk, SpillControl *spillage) { assert(chunk->len > 0 && chunk->len <= CHUNK_SZ); chunk->data = malloc(chunk->len); if (NULL == chunk->data) { perror("reread_data"); return -1; } if (0 != pread_all(chunk->spill->fd, chunk->data, chunk->len, chunk->spill_offset)) { return -1; } spillage->alloced += CHUNK_SZ; if (spillage->alloced > spillage->max_alloced) { spillage->max_alloced = spillage->alloced; } if (verbosity > 2) { if (chunk->spill->is_tmp) { fprintf(stderr, "%.3f Re-read %zd bytes (%sB) from spill file #%d\n", get_time(), chunk->len, human_size(chunk->len), chunk->spill->is_tmp); } else { fprintf(stderr, "%.3f Re-read %zd bytes (%sB) from input file\n", get_time(), chunk->len, human_size(chunk->len)); } } return 0; }
/* just gets the file SRC and store in local file DEST * doesn't parse any LIST output * returns 0 on success, else -1 */ static int do_the_get(const char *src, const char *dest, putmode_t how, unsigned opt) { char *fulldest; char *tmp; transfer_mode_t type; type = ascii_transfer(src) ? tmAscii : gvDefaultType; if(test(opt, GET_ASCII)) type = tmAscii; else if(test(opt, GET_BINARY)) type = tmBinary; tmp = getcwd(NULL, 0); if (tmp == (char *)NULL) return -1; fulldest = path_absolute(dest, tmp, 0); #if 0 && (defined(HAVE_SETPROCTITLE) || defined(linux)) if(gvUseEnvString && ftp_connected()) setproctitle("%s, get %s", ftp->url->hostname, src); #endif int r = ftp_getfile(src, dest, how, type, test(opt, GET_VERBOSE) && !gvSighupReceived && !test(opt, GET_NOHUP) ? transfer : 0); if(r == 0 && (test(opt, GET_NOHUP) || gvSighupReceived)) { fprintf(stderr, "%s [%sb of ", src, human_size(ftp->ti.size)); fprintf(stderr, "%sb]\n", human_size(ftp->ti.total_size)); } if(test(opt, GET_NOHUP)) { if(r == 0) transfer_mail_msg(_("received %s\n"), src); else transfer_mail_msg(_("failed to receive %s: %s\n"), src, ftp_getreply(false)); } free(fulldest); #if 0 && (defined(HAVE_SETPROCTITLE) || defined(linux)) if(gvUseEnvString && ftp_connected()) setproctitle("%s", ftp->url->hostname); #endif free(tmp); return r; }
static void term_handler(int signum) { time_t now = time(0); fprintf(stderr, "%s [%sB of ", ftp->ti.remote_name, human_size(ftp->ti.size)); fprintf(stderr, "%sB]\n", human_size(ftp->ti.total_size)); printf(_("SIGTERM (terminate) received, exiting...\n")); printf(_("Transfer aborted %s"), ctime(&now)); if(ftp->ti.remote_name) printf(_("%s may not have transferred correctly\n"), ftp->ti.remote_name); transfer_mail_msg(_("SIGTERM (terminate) received, exiting...\n")); transfer_mail_msg(_("Transfer aborted %s"), ctime(&now)); transfer_mail_msg(_("%s may not have transferred correctly\n"), ftp->ti.remote_name); transfer_end_nohup(); }
static int print_transfer_string(char *str, FILE *fp, transfer_info *ti, float bps, unsigned int secs, unsigned long long int eta) { int i; int len = 0; char *e = str; bool leftjust; int inescape = 0; int minlen; int number_of_dots; /* used by visual progress bar (%v) */ while(e && *e) { if(*e == '%') { leftjust = false; minlen = 0; e++; if(!*e) break; if(*e == '-') { leftjust = true; e++; } if(isdigit((int)*e)) { minlen = atoi(e); while(isdigit((int)*e)) e++; } if(leftjust) minlen = -minlen; char* sp = NULL; switch(*e) { case 'r': sp = shortpath(base_name_ptr(ti->remote_name), leftjust ? -minlen : minlen, ftp->homedir); len += max_printf(fp, minlen, "%s", sp); break; case 'R': sp = shortpath(ti->remote_name, leftjust ? -minlen : minlen, ftp->homedir); len += max_printf(fp, minlen, "%s", sp); break; case 'l': sp = shortpath(base_name_ptr(ti->local_name), leftjust ? -minlen : minlen, gvLocalHomeDir); len += max_printf(fp, minlen, "%s", sp); break; case 'L': sp = shortpath(ti->local_name, leftjust ? -minlen : minlen, gvLocalHomeDir); len += max_printf(fp, minlen, "%s", sp); break; case 's': len += max_printf(fp, minlen, "%sB", human_size(ti->size)); break; case 'S': len += max_printf(fp, minlen, "%sB", (ti->total_size == -1L ? "??" : human_size(ti->total_size))); break; case 'b': len += max_printf(fp, minlen < 2 ? minlen : minlen-2, "%sB/s", human_size(bps)); break; case 'B': if(ti->stalled >= 5) len += max_printf(fp, minlen, "%s", _("stalled")); else len += max_printf(fp, minlen < 2 ? minlen : minlen-2, "%sB/s", human_size(bps)); break; case 'e': if(eta != (unsigned) -1) len += max_printf(fp, minlen, "%s", human_time(eta)); else len += max_printf(fp, minlen, "%s", "--:--"); break; case 't': len += max_printf(fp, minlen, "%s", human_time(secs)); break; case '%': len += fprintf(fp, "%%"); break; case 'p': if(ti->total_size != -1L) len += max_printf(fp, minlen, "%.1f", (double)100*ti->size / (ti->total_size + (ti->total_size ? 0 : 1))); else len += fprintf(fp, "?"); break; case 'v': if(ti->total_size != -1L) { if(ti->total_size == ti->size) number_of_dots = minlen; else number_of_dots = (double)minlen * ti->size / (ti->total_size + 1); if(number_of_dots > minlen || number_of_dots < 0) /* just in case */ number_of_dots = minlen; i = minlen - number_of_dots; while(number_of_dots--) len += fprintf(fp, "#"); while(i--) len += fprintf(fp, " "); } else { number_of_dots = minlen / 2; i = minlen - number_of_dots; while(number_of_dots--) len += fprintf(fp, " "); if(i) { i--; len += fprintf(fp, "?"); while(i--) len += fprintf(fp, " "); } } break; case '{': inescape++; break; case '}': inescape--; break; default: len += fprintf(fp, "%%%c", *e); break; } free(sp); } else { fputc(*e, fp); if (inescape <= 0) len++; } e++; } return len; }
static ssize_t do_write(Output *output, ChunkList *chunks, Opts *options, SpillControl *spillage, int index) { ssize_t bytes = 0; Chunk *curr_chunk = output->curr_chunk; while (curr_chunk->next != NULL || output->offset < curr_chunk->len) { /* While there's something to write ... */ assert(NULL != curr_chunk->data); if (output->offset < curr_chunk->len) { /* Data available in the current Chunk */ ssize_t b; /* Send it */ do { b = write(output->fd, curr_chunk->data + output->offset, curr_chunk->len - output->offset); } while (b < 0 && EINTR == errno); if (b < 0) { /* Error */ if (EAGAIN == errno || EWOULDBLOCK == errno) break; /* Blocking is OK */ if (EPIPE == errno) return -2; /* Got EPIPE, file should be closed */ fprintf(stderr, "Error writing to %s : %s\n", output->name, strerror(errno)); return -1; } if (b == 0) break; /* Wrote nothing, try again later */ /* Update amount read */ output->written += b; output->offset += b; bytes += b; /* Record time and update linked list */ if (!output->is_reg) { output->write_time = get_time(); if (output->write_time < 0) { return -1; } while (NULL != output->next && output->next->written < output->written) { Output *n = output->next; assert(n->prev == output); pipe_list_remove(output, spillage); pipe_list_insert(output, n, spillage); } if (output == spillage->blocking_output) { spillage->blocking_output = spillage->pipe_list_head; } } } assert(output->offset <= curr_chunk->len); /* Check if at end of current Chunk */ if (output->offset == curr_chunk->len) { /* Stop sending if no more Chunks yet */ if (NULL == curr_chunk->next) break; /* Otherwise, move on to the next Chunk */ output->curr_chunk = curr_chunk->next; output->offset = 0; --curr_chunk->nwriters; if (0 != release_chunk(chunks, curr_chunk, options, spillage)) { return -1; } curr_chunk = output->curr_chunk; curr_chunk->nwriters++; if (NULL == curr_chunk->data) { /* Need to re-read spilled data */ if (0 != reread_data(curr_chunk, spillage)) return -1; } } } if (verbosity > 2 && bytes > 0) { fprintf(stderr, "%.3f Wrote %zd bytes to output #%d (%s); %lld (%sB) so far.\n", get_time(), bytes, index, output->name, (long long) output->written, human_size(output->written)); } return bytes; }
static ssize_t do_read(Opts *options, Input *in, SpillControl *spillage, ChunkList *chunks, int *read_eof, int nrefs) { ssize_t bytes; spillage->blocking_output = NULL; if (chunks->tail->len == CHUNK_SZ || NULL == chunks->tail->data) { if (spillage->alloced >= options->max) { if (options->wait_time > 0 && spillage->pipe_list_head != NULL) { double now = get_time(); if (now < 0) return -1; if (now < spillage->pipe_list_head->write_time) { /* Someone fiddled with the clock? */ spillage->pipe_list_head->write_time = now; } if (now - spillage->pipe_list_head->write_time < options->wait_time) { /* Not waited long enough, so return without reading anything */ spillage->blocking_output = spillage->pipe_list_head; return 0; } } if (0 != spill_data(options, chunks, spillage)) return -1; } if (chunks->tail->len == CHUNK_SZ) { /* Need to start a new Chunk */ if (0 != new_chunk(chunks, nrefs)) return -1; if (NULL != spillage->spill && !spillage->spill->is_tmp) { /* Set starting offset for spillage purposes */ chunks->tail->spill_offset = in->pos; chunks->tail->spill = spillage->spill; } } /* Allocate a buffer to put the data in */ chunks->tail->data = malloc(CHUNK_SZ); if (NULL == chunks->tail->data) { perror("do_read"); return -1; } spillage->alloced += CHUNK_SZ; if (spillage->alloced > spillage->max_alloced) { spillage->max_alloced = spillage->alloced; } } /* Read some data */ do { bytes = read(in->fd, chunks->tail->data + chunks->tail->len, CHUNK_SZ - chunks->tail->len); } while (bytes < 0 && errno == EINTR); if (bytes < 0) { /* Error */ if (errno == EAGAIN || errno == EWOULDBLOCK) return 0; /* Blocking is OK */ fprintf(stderr, "Error reading %s : %s\n", options->in_name, strerror(errno)); return -1; } if (bytes == 0) { /* EOF */ *read_eof = 1; if (verbosity > 2) { fprintf(stderr, "%.3f Got EOF on input\n", get_time()); } return 0; } /* Got some data, update length and in->pos */ in->pos += bytes; chunks->tail->len += bytes; if (verbosity > 2) { fprintf(stderr, "%.3f Read %zd bytes from input; %lld (%sB) so far.\n", get_time(), bytes, (long long) in->pos, human_size(in->pos)); } return 0; }
static int spill_data(Opts *options, ChunkList *chunks, SpillControl *spillage) { SpillFile *spill; Chunk *candidate; /* Look for a chunk that can be spilled */ candidate = NULL != chunks->spilled ? chunks->spilled : chunks->head; while (NULL != candidate && (candidate->nwriters > 0 || NULL == candidate->data)) { candidate = candidate->next; } if (NULL == candidate) return 0; /* Nothing suitable */ if (NULL != spillage->spill && !spillage->spill->is_tmp) { /* Input is regular, just need to forget the data read earlier */ free(candidate->data); candidate->data = NULL; spillage->alloced -= CHUNK_SZ; return 0; } /* Otherwise have to store it somewhere */ if (NULL == spillage->spill || spillage->spill->offset >= options->file_max) { spill = make_tmp_file(options, spillage->spill); if (NULL == spill) return -1; spillage->spill_file_count++; if (spillage->spill_file_count > 0) { spill->is_tmp = spillage->spill_file_count; } spillage->open_spill_files++; if (spillage->open_spill_files > spillage->max_spill_files) { spillage->max_spill_files = spillage->open_spill_files; } if (verbosity > 1) { fprintf(stderr, "%.3f Opened spill file #%d on fd %d; %d spill files now open.\n", get_time(), spillage->spill_file_count, spill->fd, spillage->open_spill_files); } } else { spill = spillage->spill; } if (0 != write_all(spill->fd, candidate->data, candidate->len)) { perror("Writing to temporary file"); return -1; } if (verbosity > 2) { fprintf(stderr, "%.3f Spilled %zd bytes (%sB) to spill file #%d\n", get_time(), candidate->len, human_size(candidate->len), spill->is_tmp); } candidate->spill = spill; candidate->spill_offset = spill->offset; spill->offset += candidate->len; spill->ref_count++; free(candidate->data); candidate->data = NULL; spillage->alloced -= CHUNK_SZ; spillage->total_spilled += candidate->len; spillage->curr_spilled += candidate->len; if (spillage->curr_spilled > spillage->max_spilled) { spillage->max_spilled = spillage->curr_spilled; } spillage->spill = spill; chunks->spilled = candidate; return 0; }
static int do_copy(Opts *options, Input *in, int noutputs, Output *outputs, int nregular, int *regular, int npipes, int *pipes, SpillControl *spillage) { ChunkList chunks = { NULL, NULL, NULL }; /* Linked list of Chunks */ struct pollfd *polls; /* structs for poll(2) */ int *poll_idx; /* indexes in outputs corresponding to entries in polls */ int *closing_pipes; /* Pipes that need to be closed */ int *closing_reg; /* Regular files that need to be closed */ int i, keeping_up = npipes, read_eof = 0, nclosed = 0; chunks.head = chunks.tail = calloc(1, sizeof(Chunk)); /* Initial Chunk */ polls = malloc((noutputs + 1) * sizeof(struct pollfd)); poll_idx = malloc((noutputs + 1) * sizeof(int)); closing_pipes = malloc((npipes + 1) * sizeof(int)); closing_reg = malloc((nregular + 1) * sizeof(int)); if (NULL == chunks.head || NULL == polls || NULL == poll_idx || NULL == closing_pipes || NULL == closing_reg) { perror("do_copy"); return -1; } chunks.head->ref_count = noutputs; chunks.head->nwriters = noutputs; /* Point all outputs to the initial Chunk and build linked list of pipes */ for (i = 0; i < noutputs; i++) { outputs[i].curr_chunk = chunks.head; if (!outputs[i].is_reg) pipe_list_push(&outputs[i], spillage); } do { /* Main loop */ int npolls = 0, pipe_close = 0, reg_close = 0; /* Are we waiting for a slow output? */ int blocked = (NULL != spillage->blocking_output && spillage->alloced >= options->max); int timeout = -1; int should_read; if (blocked) { /* Work out how long to wait in poll */ double now = get_time(); double left; if (now < 0) return -1; if (now < spillage->blocking_output->write_time) { /* Someone fiddled with the clock? */ spillage->blocking_output->write_time = now; } left = options->wait_time - (now - spillage->blocking_output->write_time); timeout = left > 0 ? (int)(left * 1000) : 0; if (timeout == 0) { /* Remove the block */ blocked = 0; spillage->blocking_output = NULL; } } /* Only try to read if not at EOF; either there are no pipes or at least one pipe has nothing left to write; and we aren't waiting for a slow output in order to prevent spillage */ should_read = (!read_eof && (npipes == 0 || keeping_up > 0) && !blocked); if (should_read) { if (in->reg) { /* If reading a regular file, do it now */ if (0 != do_read(options, in, spillage, &chunks, &read_eof, noutputs - nclosed)) { return -1; } } else { /* Otherwise add it to the poll list */ polls[npolls].fd = in->fd; poll_idx[npolls] = -1; polls[npolls].events = POLLIN; polls[npolls++].revents = 0; } } keeping_up = 0; /* Number of pipes that are keeping up */ /* Add all the pipe outputs that have something to write to the poll list */ for (i = 0; i < npipes; i++) { if (outputs[pipes[i]].curr_chunk != chunks.tail || outputs[pipes[i]].offset < chunks.tail->len) { /* Something to write */ polls[npolls].fd = outputs[pipes[i]].fd; poll_idx[npolls] = i; polls[npolls].events = POLLOUT|POLLERR|POLLHUP; polls[npolls++].revents = 0; } else { /* Keeping up or finished */ if (read_eof) { closing_pipes[pipe_close++] = i; timeout = 0; /* Ensure pipe gets closed promptly */ } else { keeping_up++; } } } if (npolls > 0) { /* Need to do some polling */ int ready; do { ready = poll(polls, npolls, timeout); } while (ready < 0 && EINTR == errno); if (ready < 0) { perror("poll failed in do_copy"); return -1; } for (i = 0; i < npolls && ready > 0; i++) { if (polls[i].revents) { /* Got some events */ --ready; if (poll_idx[i] < 0) { /* Input, try to read from it. */ if (0 != do_read(options, in, spillage, &chunks, &read_eof, noutputs - nclosed)) { return -1; } } else { /* Output, try to write to it. */ Output *output = &outputs[pipes[poll_idx[i]]]; ssize_t res = do_write(output, &chunks, options, spillage, pipes[poll_idx[i]]); if (-2 == res) { /* Got EPIPE, add to closing_pipes list */ closing_pipes[pipe_close++] = poll_idx[i]; continue; } else if (res < 0) { /* other write error, give up */ return -1; } if (output->curr_chunk == chunks.tail && output->offset == chunks.tail->len) { /* All the data so far has been written to this output */ if (read_eof) { /* If finished reading, add to closing_pipes */ closing_pipes[pipe_close++] = poll_idx[i]; } else { /* otherwise, add to keeping_up count, to encourage more reading */ keeping_up++; } } } } } } /* End of polling section */ /* Deal with regular output files */ for (i = 0; i < nregular; i++) { /* Try to write */ if (do_write(&outputs[regular[i]], &chunks, options, spillage, regular[i]) < 0) { return -1; } if (read_eof && outputs[regular[i]].curr_chunk == chunks.tail && outputs[regular[i]].offset == chunks.tail->len) { /* If all data written and finished reading, add to closing_reg list */ closing_reg[reg_close++] = i; } } /* Close any regular files that have finished */ for (i = reg_close - 1; i >= 0; i--) { int to_close = regular[closing_reg[i]]; assert(outputs[to_close].fd >= 0); if (0 != close(outputs[to_close].fd)) { fprintf(stderr, "Error closing %s : %s\n", outputs[to_close].name, strerror(errno)); return -1; } if (verbosity > 1) { fprintf(stderr, "%.3f Closed output #%d (%s)\n", get_time(), to_close, outputs[to_close].name); } outputs[to_close].fd = -1; /* Remove the entry from the regular array */ if (closing_reg[i] < nregular - 1) { memmove(®ular[closing_reg[i]], ®ular[closing_reg[i] + 1], (nregular - closing_reg[i] - 1) * sizeof(regular[0])); } if (0 != release_chunks(&chunks, outputs[to_close].curr_chunk, options, spillage)) { return -1; } nclosed++; nregular--; } /* Close any poll-able files that have finished */ for (i = pipe_close - 1; i >= 0; i--) { int to_close = pipes[closing_pipes[i]]; assert(outputs[to_close].fd >= 0); if (0 != close(outputs[to_close].fd)) { fprintf(stderr, "Error closing %s : %s\n", outputs[to_close].name, strerror(errno)); return -1; } if (verbosity > 1) { fprintf(stderr, "%.3f Closed output #%d (%s)\n", get_time(), to_close, outputs[to_close].name); } outputs[to_close].fd = -1; /* Remove the entry from the pipes array */ if (closing_pipes[i] < npipes - 1) { memmove(&pipes[closing_pipes[i]], &pipes[closing_pipes[i] + 1], (npipes - closing_pipes[i] - 1) * sizeof(pipes[0])); } /* Remove from spillage linked list */ pipe_list_remove(&outputs[to_close], spillage); /* Release any data referenced by this output */ if (0 != release_chunks(&chunks, outputs[to_close].curr_chunk, options, spillage)) { return -1; } nclosed++; npipes--; } } while (nclosed < noutputs); if (verbosity > 0) { double now = get_time(); fprintf(stderr, "%.3f Read %lld bytes (%sB) from input (%s)\n", now, (long long) in->pos, human_size(in->pos), options->in_name); for (i = 0; i < noutputs; i++) { fprintf(stderr, "%.3f Wrote %lld bytes (%sB) to output #%d (%s)\n", now, (long long) outputs[i].written, human_size(outputs[i].written), i, outputs[i].name); } fprintf(stderr, "%.3f Maximum buffer used = %zd bytes (%sB)\n", now, spillage->max_alloced, human_size(spillage->max_alloced)); if (!in->reg) { fprintf(stderr, "%.3f Spilled %lld bytes (%sB) to temporary files\n", now, (long long) spillage->total_spilled, human_size(spillage->total_spilled)); if (spillage->total_spilled > 0) { fprintf(stderr, "%.3f Maximum spilled at one time = %lld bytes (%sB)\n", now, (long long) spillage->max_spilled, human_size(spillage->max_spilled)); fprintf(stderr, "%.3f Total temporary files opened = %d\n", now, spillage->spill_file_count); fprintf(stderr, "%.3f Maximum temporary files in use at one time = %d\n", now, spillage->max_spill_files); } } } return 0; }
int ExamineBitmap(char *filename, int brief, struct supertype *st) { /* * Read the bitmap file and display its contents */ bitmap_super_t *sb; bitmap_info_t *info; int rv = 1; char buf[64]; info = bitmap_file_read(filename, brief, &st); if (!info) return rv; sb = &info->sb; printf(" Filename : %s\n", filename); printf(" Magic : %08x\n", sb->magic); if (sb->magic != BITMAP_MAGIC) { fprintf(stderr, Name ": invalid bitmap magic 0x%x, the bitmap file appears to be corrupted\n", sb->magic); } printf(" Version : %d\n", sb->version); if (sb->version < BITMAP_MAJOR_LO || sb->version > BITMAP_MAJOR_HI) { fprintf(stderr, Name ": unknown bitmap version %d, either the bitmap file is corrupted or you need to upgrade your tools\n", sb->version); goto free_info; } rv = 0; if (st && st->ss->swapuuid) { printf(" UUID : %08x.%08x.%08x.%08x\n", swapl(*(__u32 *)(sb->uuid+0)), swapl(*(__u32 *)(sb->uuid+4)), swapl(*(__u32 *)(sb->uuid+8)), swapl(*(__u32 *)(sb->uuid+12))); } else { printf(" UUID : %08x.%08x.%08x.%08x\n", *(__u32 *)(sb->uuid+0), *(__u32 *)(sb->uuid+4), *(__u32 *)(sb->uuid+8), *(__u32 *)(sb->uuid+12)); } printf(" Events : %llu\n", (unsigned long long)sb->events); printf(" Events Cleared : %llu\n", (unsigned long long)sb->events_cleared); printf(" State : %s\n", bitmap_state(sb->state)); printf(" Chunksize : %s\n", human_chunksize(sb->chunksize)); printf(" Daemon : %ds flush period\n", sb->daemon_sleep); if (sb->write_behind) sprintf(buf, "Allow write behind, max %d", sb->write_behind); else sprintf(buf, "Normal"); printf(" Write Mode : %s\n", buf); printf(" Sync Size : %llu%s\n", (unsigned long long)sb->sync_size/2, human_size(sb->sync_size * 512)); if (brief) goto free_info; printf(" Bitmap : %llu bits (chunks), %llu dirty (%2.1f%%)\n", info->total_bits, info->dirty_bits, 100.0 * info->dirty_bits / (info->total_bits + 1)); free_info: free(info); return rv; }