static void queue_work_item(const char *url, char *bytes, int size, bool request_failed, bool tried_server) { volatile struct work_item *item = alloc_work_item(url, bytes, size, request_failed, tried_server); // add the work item to the work list pthread_mutex_lock(&m_worker_mutex); LIST_PUSH(m_work_items, item); pthread_cond_signal(&m_worker_cond); pthread_mutex_unlock(&m_worker_mutex); }
void distribute(int cmd_argc, char **cmd_argv) { FILE *origin_fp = NULL; long parity_count = -1; List *partials = new_list(); List *paths = new_list(); // Store path to create once. // Parse options. int i = 1; // cmd_argv[0] is always command name, i.e. "distribute." while (i < cmd_argc) { // Parse path to original file. if (strcmp("-f", cmd_argv[i]) == 0) { if (++i >= cmd_argc) { help(cmd_argv[0]); } origin_fp = dv_fopen(cmd_argv[i], "r"); } // Parse parity count else if (strcmp("-p", cmd_argv[i]) == 0) { if (++i >= cmd_argc) { help(cmd_argv[0]); } char *endptr; parity_count = strtol(cmd_argv[i], &endptr, 0); if (errno || *endptr != '\0') { help(cmd_argv[0]); } } // Parse path to partial files. else { Str *s = new_str(cmd_argv[i]); LIST_PUSH(paths, s); } i++; } // Check options if (parity_count < 0 || parity_count > 1) { dv_error("Option -p is required and it must be 0 or 1."); } if (origin_fp == NULL) { dv_error("Option -f is required."); } long split_count = paths->length - parity_count; if (split_count <= 0) { dv_error("Too few distributed paths are specified."); } // Create partials LIST_REWIND(paths); for (long i=0; LIST_HAS_NEXT(paths); i++) { char *path = GET_STR(LIST_NEXT(paths)); Partial *p = new_partial(split_count, i, path); push_cleanup_list(path); LIST_PUSH(partials, p); } do_distribute(origin_fp, partials); // Cleanup if (origin_fp) { dv_fclose(origin_fp); } DV_DEC_REF(partials); DV_DEC_REF(paths); }
void do_distribute(FILE *origin_fp, List *partials) { long split_count = ((Partial *)LIST_GET(partials, 0))->split_count; int parity_count = partials->length - split_count; assert(parity_count == 0 || parity_count == 1); // Separate partials into splits and parities. List *splits = new_list(); Partial *parity = NULL; LIST_REWIND(partials); for (long i=0; i<split_count; i++) { Partial *p = (Partial *)LIST_NEXT(partials); assert(p->split_count == split_count); assert(p->index == i); DV_INC_REF(p); LIST_PUSH(splits, p); } if (parity_count == 1) { Partial *p = (Partial *)LIST_NEXT(partials); assert(p->split_count == split_count); assert(p->index == split_count); parity = p; } long buff_size = 1024; char buffer[buff_size]; // write conteent_length header; long content_length = dv_file_size(origin_fp); long header_size = snprintf(buffer, buff_size, "content_length: %ld\n\n", content_length); if (header_size == buff_size) { dv_error("The buffer size is too short to create content_length header."); } LIST_REWIND(splits); for (long i=0; i<header_size; i++) { // Split char into bits and write. for (int j=0; j<8; j++) { if (! LIST_HAS_NEXT(splits)) { LIST_REWIND(splits); } PARTIAL_PUTB(LIST_NEXT(splits), buffer[i] & (1 << j)); } } // Read from origin_fp and write to splits. int c; while ((c = dv_fgetc(origin_fp)) != EOF) { // Split char into bits and write for (int i=0; i<8; i++) { if (! (LIST_HAS_NEXT(splits))) { LIST_REWIND(splits); } PARTIAL_PUTB(LIST_NEXT(splits), c & (1 << i)); } } // Create parity if necesary. if (parity) { take_parity(splits, parity); } }