int blk_verify_fingerprint(uint64_t fingerprint, char *data, size_t length) { win_reset(); memcpy(gbuf, data, length); gbuf_end=gbuf+length; gcp=gbuf; blk_free(&blk); if(!blk && !(blk=blk_alloc())) return -1; blk->length=0; blk->fingerprint=0; // FIX THIS: blk_read should return 1 when it has a block. // But, if the block is too small (because the end of the file // happened), it returns 0, and blks_generate treats it as having found // a final block. // So, here the return of blk_read is ignored and we look at the // position of gcp instead. blk_read(); //printf("%d %d\n", blk->length, length); //printf("%016"PRIX64" %016"PRIX64" ", // blk->fingerprint, fingerprint); //printf("%s\n", blk->fingerprint==fingerprint?"yes":"no"); if(gcp==gbuf_end && blk->fingerprint==fingerprint) return 1; return 0; }
// The server uses this for verification. int blk_read_verify(struct blk *blk_to_verify) { if(!win) { rconf_init(&rconf); if(!(win=win_alloc(&rconf))) return -1; } gbuf=blk_to_verify->data; gbuf_end=gbuf+blk_to_verify->length; gcp=gbuf; if(!blk && !(blk=blk_alloc())) return -1; blk->length=0; blk->fingerprint=0; // FIX THIS: blk_read should return 1 when it has a block. // But, if the block is too small (because the end of the file // happened), it returns 0, and blks_generate treats it as having found // a final block. // So, here the return of blk_read is ignored and we look at the // position of gcp instead. blk_read(); if(gcp==gbuf_end && blk->fingerprint==blk_to_verify->fingerprint) return 1; return 0; }
// Return -1 for error, 0 for entry not changed, 1 for entry changed (or new). static int entry_changed(struct sbuf *sb, struct manios *manios, struct asfd *chfd, struct sbuf **csb) { static int finished=0; static struct blk *blk=NULL; if(finished) return 1; if((*csb)->path.buf) { // Already have an entry. } else { // Need to read another. if(!blk && !(blk=blk_alloc())) return -1; switch(manio_read_with_blk(manios->current, *csb, blk, NULL)) { case 1: // Reached the end. sbuf_free(csb); blk_free(&blk); finished=1; return 1; case -1: return -1; } if(!(*csb)->path.buf) { logp("Should have a path at this point, but do not, in %s\n", __func__); return -1; } // Got an entry. } while(1) { switch(sbuf_pathcmp(*csb, sb)) { case 0: return found_in_current_manifest(*csb, sb, manios, &blk, chfd); case 1: return 1; case -1: // Behind - need to read more data from the old // manifest. switch(manio_read_with_blk(manios->current, *csb, blk, NULL)) { case 1: // Reached the end. sbuf_free(csb); blk_free(&blk); return 1; case -1: return -1; } // Got something, go back around the loop. } } return 0; }
static void add_blk_and_data_files(struct blist *blist, uint64_t save_path) { struct blk *blk; fail_unless((blk=blk_alloc())!=NULL); blk->fingerprint=prng_next64(); prng_md5sum(blk->md5sum); blk->savepath=save_path; blist_add_blk(blist, blk); }
static void add_blk(struct blist *blist) { struct blk *blk; fail_unless((blk=blk_alloc())!=NULL); blk->fingerprint=prng_next64(); prng_md5sum(blk->md5sum); blk->savepath=prng_next64(); blist_add_blk(blist, blk); }
static void* hashstr_alloc(void **b, int n, uint x) { struct hashstrblk *m = 0; void *r[2] = { 0, 0 }; if(!b[0]) b[0] = blk_create(x); if(n > 0) if(m = blk_alloc(b[0], sizeof(int) + n * sizeof(char*))) { m->n = n - 1; memset(m->h, 0, n << 2); b[1] = m; } return (void*)m; }
struct blk *blk_alloc_with_data(uint32_t max_data_length) { struct blk *blk=NULL; if(!(blk=blk_alloc())) return NULL; if((blk->data=(char *) calloc_w(1, sizeof(char)*max_data_length, __func__))) return blk; blk_free(&blk); return NULL; }
// When a backup is ongoing, use this to add newly complete candidates. int candidate_add_fresh(const char *path, struct conf *conf) { int ars; int ret=-1; gzFile zp=NULL; const char *cp=NULL; struct sbuf *sb=NULL; struct candidate *candidate=NULL; struct blk *blk=NULL; if(!(candidate=candidates_add_new())) goto end; cp=path+strlen(conf->directory); while(cp && *cp=='/') cp++; if(!(candidate->path=strdup(cp))) { log_out_of_memory(__func__); goto end; } if(!(sb=sbuf_alloc(conf)) || !(blk=blk_alloc()) || !(zp=gzopen_file(path, "rb"))) goto end; while(zp) { if((ars=sbuf_fill_from_gzfile(sb, NULL /* struct async */, zp, blk, NULL, conf))<0) goto end; else if(ars>0) { // Reached the end. break; } if(!*(blk->weak)) continue; if(is_hook(blk->weak)) { if(sparse_add_candidate(blk->weak, candidate)) goto end; } *blk->weak='\0'; } if(scores_grow(scores, candidates_len)) goto end; candidates_set_score_pointers(candidates, candidates_len, scores); scores_reset(scores); //printf("HERE: %d candidates\n", (int)candidates_len); ret=0; end: gzclose_fp(&zp); sbuf_free(sb); blk_free(blk); return ret; }
static int write_to_dpth(struct dpth *dpth, const char *savepathstr) { int ret; struct iobuf wbuf; struct blk *blk=blk_alloc(); savepathstr_to_bytes(savepathstr, blk->savepath); wbuf.buf=strdup_w("abc", __FUNCTION__); wbuf.len=3; ret=dpth_protocol2_fwrite(dpth, &wbuf, blk); free_w(&wbuf.buf); blk_free(&blk); return ret; }
void hash_add_elem_idx(Hash* hash, const Elem* elem, int idx) { HElem* he = blk_alloc(sizeof(*he)); unsigned int hcode; assert(hash_is_valid(hash)); assert(elem_is_valid(elem)); assert(he != NULL); hcode = elem_hash(elem) % hash->size; he->value.elem_idx.elem = elem; he->value.elem_idx.idx = idx; he->next = hash->bucket[hcode]; hash->bucket[hcode] = he; hash->elems++; }
void hash_add_numb(Hash* hash, const Numb* numb) { HElem* he = blk_alloc(sizeof(*he)); unsigned int hcode; assert(hash_is_valid(hash)); assert(numb_is_valid(numb)); assert(hash->type == HASH_NUMB); assert(he != NULL); hcode = numb_hash(numb) % hash->size; he->value.numb = numb; he->next = hash->bucket[hcode]; hash->bucket[hcode] = he; hash->elems++; }
void hash_add_tuple(Hash* hash, const Tuple* tuple) { HElem* he = blk_alloc(sizeof(*he)); unsigned int hcode; assert(hash_is_valid(hash)); assert(tuple_is_valid(tuple)); assert(hash->type == HASH_TUPLE); assert(he != NULL); hcode = tuple_hash(tuple) % hash->size; he->value.tuple = tuple; he->next = hash->bucket[hcode]; hash->bucket[hcode] = he; hash->elems++; }
void hash_add_mono(Hash* hash, const Mono* mono) { HElem* he = blk_alloc(sizeof(*he)); unsigned int hcode; assert(hash_is_valid(hash)); assert(mono_is_valid(mono)); assert(hash->type == HASH_MONO); assert(he != NULL); hcode = mono_hash(mono) % hash->size; he->value.mono = mono; he->next = hash->bucket[hcode]; hash->bucket[hcode] = he; hash->elems++; }
static void list_add_data(List* list, const ListData* data) { ListElem* elem = blk_alloc(sizeof(*elem)); assert(list_is_valid(list)); assert(elem != NULL); assert(data != NULL); elem->data = *data; elem->next = &list->anchor; elem->prev = list->anchor.prev; list->anchor.prev->next = elem; list->anchor.prev = elem; list->elems++; }
void hash_add_entry(Hash* hash, const Entry* entry) { HElem* he = blk_alloc(sizeof(*he)); const Tuple* tuple; unsigned int hcode; assert(hash_is_valid(hash)); assert(entry_is_valid(entry)); assert(hash->type == HASH_ENTRY); assert(he != NULL); tuple = entry_get_tuple(entry); hcode = tuple_hash(tuple) % hash->size; he->value.entry = entry; he->next = hash->bucket[hcode]; hash->bucket[hcode] = he; hash->elems++; }
static #endif int champ_server_deal_with_rbuf_sig(struct asfd *asfd, const char *directory, struct scores *scores) { struct blk *blk; if(!(blk=blk_alloc())) return -1; blist_add_blk(asfd->blist, blk); if(!asfd->blist->blk_to_dedup) asfd->blist->blk_to_dedup=blk; if(blk_set_from_iobuf_sig(blk, asfd->rbuf)) return -1; //logp("Got fingerprint from %d: %lu - %lu\n", // asfd->fd, blk->index, blk->fingerprint); return deduplicate_maybe(asfd, blk, directory, scores); }
static void wb_init(struct write_block *wb , struct block *b) { if (b==NULL) { wb->head = blk_alloc(); wb->len = 0; wb->current = wb->head; wb->ptr = 0; wb_push(wb, &wb->len, sizeof(wb->len)); } else { wb->head = b; int * plen = (int *)b->buffer; int sz = *plen; wb->len = sz; while (b->next) { sz -= BLOCK_SIZE; b = b->next; } wb->current = b; wb->ptr = sz; } }
inline static void wb_push(struct write_block *b, const void *buf, int sz) { const char * buffer = buf; if (b->ptr == BLOCK_SIZE) { _again: b->current = b->current->next = blk_alloc(); b->ptr = 0; } if (b->ptr <= BLOCK_SIZE - sz) { memcpy(b->current->buffer + b->ptr, buffer, sz); b->ptr+=sz; b->len+=sz; } else { int copy = BLOCK_SIZE - b->ptr; memcpy(b->current->buffer + b->ptr, buffer, copy); buffer += copy; b->len += copy; sz -= copy; goto _again; } }
static uint hashstr_add(void **b, uint p, uchar *s, uint n) { uint a, i = 0, h = 0; struct hashstrblk *x; struct hashstritem *m; if(!b || !(x = b[1])) return p; for(;;) { if(i >= n) break; a = s[i]; if(n == -1 && !a) break; i++; h = h * 25 + a; } n = h & x->n; m = x->h[n]; while(m) { if(m->h == h && m->i == i) if(!memcmp(s, m->s, i)) return m->p; m = m->n; } if(m = blk_alloc(b[0], sizeof(void*) + 3 * sizeof(int) + i * sizeof(char))) { m->h = h; m->i = i; m->p = p; memcpy(m->s, s, i); m->n = x->h[n]; x->h[n] = m; } return p; }
static int add_to_sig_list(struct slist *slist, struct iobuf *rbuf) { // Goes on slist->add_sigs_here struct blk *blk; struct protocol2 *protocol2; if(!(blk=blk_alloc())) return -1; blist_add_blk(slist->blist, blk); protocol2=slist->add_sigs_here->protocol2; if(!protocol2->bstart) protocol2->bstart=blk; if(!protocol2->bsighead) protocol2->bsighead=blk; if(blk_set_from_iobuf_sig(blk, rbuf)) return -1; // Need to send sigs to champ chooser, therefore need to point // to the oldest unsent one if nothing is pointed to yet. if(!slist->blist->blk_for_champ_chooser) slist->blist->blk_for_champ_chooser=blk; return 0; }
static int add_to_sig_list(struct slist *slist, struct blist *blist, struct iobuf *rbuf, struct dpth *dpth, struct conf *conf) { // Goes on slist->add_sigs_here struct blk *blk; struct burp2 *burp2; if(!(blk=blk_alloc())) return -1; blist_add_blk(blist, blk); burp2=slist->add_sigs_here->burp2; if(!burp2->bstart) burp2->bstart=blk; if(!burp2->bsighead) burp2->bsighead=blk; if(split_sig(rbuf->buf, rbuf->len, &blk->fingerprint, blk->md5sum)) return -1; // Need to send sigs to champ chooser, therefore need to point // to the oldest unsent one if nothing is pointed to yet. if(!blist->blk_for_champ_chooser) blist->blk_for_champ_chooser=blk; return 0; }
// This is basically backup_phase3_server() from protocol1. It used to merge the // unchanged and changed data into a single file. Now it splits the manifests // into several files. int backup_phase3_server_protocol2(struct sdirs *sdirs, struct conf **confs) { int ret=1; int pcmp=0; char *hooksdir=NULL; char *dindexdir=NULL; char *manifesttmp=NULL; struct sbuf *usb=NULL; struct sbuf *csb=NULL; struct blk *blk=NULL; int finished_ch=0; int finished_un=0; struct manio *newmanio=NULL; struct manio *chmanio=NULL; struct manio *unmanio=NULL; uint64_t fcount=0; logp("Start phase3\n"); if(!(manifesttmp=get_tmp_filename(sdirs->rmanifest)) || !(newmanio=manio_alloc()) || !(chmanio=manio_alloc()) || !(unmanio=manio_alloc()) || !(hooksdir=prepend_s(manifesttmp, "hooks")) || !(dindexdir=prepend_s(manifesttmp, "dindex")) || manio_init_write(newmanio, manifesttmp) || manio_init_write_hooks(newmanio, get_string(confs[OPT_DIRECTORY]), hooksdir, sdirs->rmanifest) || manio_init_write_dindex(newmanio, dindexdir) || manio_init_read(chmanio, sdirs->changed) || manio_init_read(unmanio, sdirs->unchanged) || !(usb=sbuf_alloc(confs)) || !(csb=sbuf_alloc(confs))) goto end; while(!finished_ch || !finished_un) { if(!blk && !(blk=blk_alloc())) goto end; if(!finished_un && usb && !usb->path.buf) { switch(manio_sbuf_fill(unmanio, NULL /* no async */, usb, NULL, NULL, confs)) { case -1: goto end; case 1: finished_un++; } } if(!finished_ch && csb && !csb->path.buf) { switch(manio_sbuf_fill(chmanio, NULL /* no async */, csb, NULL, NULL, confs)) { case -1: goto end; case 1: finished_ch++; } } if((usb && usb->path.buf) && (!csb || !csb->path.buf)) { switch(manio_copy_entry(NULL /* no async */, &usb, usb, &blk, unmanio, newmanio, confs)) { case -1: goto end; case 1: finished_un++; } } else if((!usb || !usb->path.buf) && (csb && csb->path.buf)) { switch(manio_copy_entry(NULL /* no async */, &csb, csb, &blk, chmanio, newmanio, confs)) { case -1: goto end; case 1: finished_ch++; } } else if((!usb || !usb->path.buf) && (!csb || !(csb->path.buf))) { continue; } else if(!(pcmp=sbuf_pathcmp(usb, csb))) { // They were the same - write one. switch(manio_copy_entry(NULL /* no async */, &csb, csb, &blk, chmanio, newmanio, confs)) { case -1: goto end; case 1: finished_ch++; } } else if(pcmp<0) { switch(manio_copy_entry(NULL /* no async */, &usb, usb, &blk, unmanio, newmanio, confs)) { case -1: goto end; case 1: finished_un++; } } else { switch(manio_copy_entry(NULL /* no async */, &csb, csb, &blk, chmanio, newmanio, confs)) { case -1: goto end; case 1: finished_ch++; } } } fcount=newmanio->fcount; // Flush to disk and set up for reading. if(manio_free(&newmanio) || !(newmanio=manio_alloc()) || manio_init_read(newmanio, sdirs->rmanifest)) goto end; // Rename race condition should be of no consequence here, as the // manifest should just get recreated automatically. if(do_rename(manifesttmp, sdirs->rmanifest)) goto end; else { recursive_delete(sdirs->changed, NULL, 1); recursive_delete(sdirs->unchanged, NULL, 1); } if(sparse_generation(newmanio, fcount, sdirs, confs)) goto end; ret=0; logp("End phase3\n"); end: manio_free(&newmanio); manio_free(&chmanio); manio_free(&unmanio); sbuf_free(&csb); sbuf_free(&usb); blk_free(&blk); free_w(&hooksdir); free_w(&dindexdir); free_w(&manifesttmp); return ret; }
// This deals with reading in the sparse index, as well as actual candidate // manifests. enum cand_ret candidate_load(struct candidate *candidate, const char *path, struct scores *scores) { enum cand_ret ret=CAND_RET_PERM; struct fzp *fzp=NULL; struct sbuf *sb=NULL; struct blk *blk=NULL; if(!(sb=sbuf_alloc(PROTO_2)) || !(blk=blk_alloc())) { ret=CAND_RET_PERM; goto error; } if(!(fzp=fzp_gzopen(path, "rb"))) { ret=CAND_RET_TEMP; goto error; } while(fzp) { sbuf_free_content(sb); switch(sbuf_fill_from_file(sb, fzp, blk, NULL)) { case 1: goto end; case -1: logp("Error reading %s in %s, pos %d\n", path, __func__, fzp_tell(fzp)); ret=CAND_RET_TEMP; goto error; } if(blk_fingerprint_is_hook(blk)) { if(sparse_add_candidate(&blk->fingerprint, candidate)) { ret=CAND_RET_PERM; goto error; } } else if(sb->path.cmd==CMD_MANIFEST) { if(!(candidate=candidates_add_new())) { ret=CAND_RET_PERM; goto error; } candidate->path=sb->path.buf; sb->path.buf=NULL; } blk->fingerprint=0; } end: if(scores_grow(scores, candidates_len)) { ret=CAND_RET_PERM; goto error; } candidates_set_score_pointers(candidates, candidates_len, scores); scores_reset(scores); //logp("Now have %d candidates\n", (int)candidates_len); ret=CAND_RET_OK; error: fzp_close(&fzp); sbuf_free(&sb); blk_free(&blk); return ret; }
int do_restore_client(struct asfd *asfd, struct conf **confs, enum action act, int vss_restore) { int ret=-1; char msg[512]=""; struct sbuf *sb=NULL; struct blk *blk=NULL; BFILE *bfd=NULL; char *fullpath=NULL; char *style=NULL; char *datpath=NULL; struct cntr *cntr=get_cntr(confs); enum protocol protocol=get_protocol(confs); int strip=get_int(confs[OPT_STRIP]); int overwrite=get_int(confs[OPT_OVERWRITE]); const char *backup=get_string(confs[OPT_BACKUP]); const char *regex=get_string(confs[OPT_REGEX]); const char *restore_prefix=get_string(confs[OPT_RESTOREPREFIX]); const char *encryption_password=get_string(confs[OPT_ENCRYPTION_PASSWORD]); if(!(bfd=bfile_alloc())) goto end; bfile_init(bfd, 0, cntr); snprintf(msg, sizeof(msg), "%s %s:%s", act_str(act), backup?backup:"", regex?regex:""); logp("doing %s\n", msg); if(asfd->write_str(asfd, CMD_GEN, msg) || asfd_read_expect(asfd, CMD_GEN, "ok")) goto error; logp("doing %s confirmed\n", act_str(act)); #if defined(HAVE_WIN32) if(act==ACTION_RESTORE) win32_enable_backup_privileges(); #endif if(!(style=get_restore_style(asfd, confs))) goto error; if(!strcmp(style, RESTORE_SPOOL)) { if(restore_spool(asfd, confs, &datpath)) goto error; } else logp("Streaming restore direct\n"); logf("\n"); if(get_int(confs[OPT_SEND_CLIENT_CNTR]) && cntr_recv(asfd, confs)) goto error; if(!(sb=sbuf_alloc(protocol)) || (protocol==PROTO_2 && !(blk=blk_alloc()))) { log_and_send_oom(asfd, __func__); goto error; } while(1) { sbuf_free_content(sb); if(protocol==PROTO_1) sb->flags |= SBUF_CLIENT_RESTORE_HACK; switch(sbuf_fill_from_net(sb, asfd, blk, datpath, cntr)) { case 0: break; case 1: if(asfd->write_str(asfd, CMD_GEN, "restoreend ok")) goto error; goto end; // It was OK. default: case -1: goto error; } if(protocol==PROTO_2) { if(blk->data) { int wret=0; if(act==ACTION_VERIFY) cntr_add(cntr, CMD_DATA, 1); else wret=write_data(asfd, bfd, blk); if(!datpath) blk_free_content(blk); blk->data=NULL; if(wret) goto error; continue; } else if(sb->endfile.buf) { continue; } } switch(sb->path.cmd) { case CMD_DIRECTORY: case CMD_FILE: case CMD_ENC_FILE: case CMD_SOFT_LINK: case CMD_HARD_LINK: case CMD_SPECIAL: case CMD_METADATA: case CMD_ENC_METADATA: case CMD_VSS: case CMD_ENC_VSS: case CMD_VSS_T: case CMD_ENC_VSS_T: case CMD_EFS_FILE: if(strip) { int s; s=strip_path_components(asfd, sb, strip, cntr, protocol); if(s<0) goto error; if(s==0) { // Too many components stripped // - carry on. continue; } // It is OK, sb.path is now stripped. } free_w(&fullpath); if(!(fullpath=prepend_s(restore_prefix, sb->path.buf))) { log_and_send_oom(asfd, __func__); goto error; } if(act==ACTION_RESTORE) { strip_invalid_characters(&fullpath); if(!overwrite_ok(sb, overwrite, #ifdef HAVE_WIN32 bfd, #endif fullpath)) { char msg[512]=""; // Something exists at that path. snprintf(msg, sizeof(msg), "Path exists: %s\n", fullpath); if(restore_interrupt(asfd, sb, msg, cntr, protocol)) goto error; continue; } } break; case CMD_MESSAGE: case CMD_WARNING: log_recvd(&sb->path, cntr, 1); logf("\n"); continue; default: break; } switch(sb->path.cmd) { // These are the same in both protocol1 and protocol2. case CMD_DIRECTORY: if(restore_dir(asfd, sb, fullpath, act, cntr, protocol)) goto error; continue; case CMD_SOFT_LINK: case CMD_HARD_LINK: if(restore_link(asfd, sb, fullpath, act, cntr, protocol, restore_prefix)) goto error; continue; case CMD_SPECIAL: if(restore_special(asfd, sb, fullpath, act, cntr, protocol)) goto error; continue; default: break; } if(protocol==PROTO_2) { if(restore_switch_protocol2(asfd, sb, fullpath, act, bfd, vss_restore, cntr)) goto error; } else { if(restore_switch_protocol1(asfd, sb, fullpath, act, bfd, vss_restore, cntr, encryption_password)) goto error; } } end: ret=0; error: // It is possible for a fd to still be open. bfd->close(bfd, asfd); bfile_free(&bfd); cntr_print_end(cntr); cntr_print(cntr, act); if(!ret) logp("%s finished\n", act_str(act)); else logp("ret: %d\n", ret); sbuf_free(&sb); free_w(&style); if(datpath) { recursive_delete(datpath); free_w(&datpath); } free_w(&fullpath); blk_free(&blk); return ret; }
// The client uses this. // Return 0 for OK. 1 for OK, and file ended, -1 for error. int blks_generate(struct asfd *asfd, struct conf **confs, struct sbuf *sb, struct blist *blist, int just_opened) { static ssize_t bytes; first=just_opened; if(!blk && !(blk=blk_alloc_with_data(rconf.blk_max))) return -1; if(gcp<gbuf_end) { // Could have got a fill before buf ran out - // need to resume from the same place in that case. if(blk_read_to_list(sb, blist)) return 0; // Got a block. // Did not get a block. Carry on and read more. } while((bytes=rabin_read(sb, gbuf, rconf.blk_max))) { gcp=gbuf; gbuf_end=gbuf+bytes; sb->protocol2->bytes_read+=bytes; if(blk_read_to_list(sb, blist)) return 0; // Got a block // Did not get a block. Maybe should try again? // If there are async timeouts, look at this! return 0; } // Getting here means there is no more to read from the file. // Make sure to deal with anything left over. if(!sb->protocol2->bytes_read) { // Empty file, set up an empty block so that the server // can skip over it. if(!(sb->protocol2->bstart=blk_alloc())) return -1; sb->protocol2->bsighead=blk; blist_add_blk(blist, blk); blk=NULL; } else if(blk) { if(blk->length) { if(first) { sb->protocol2->bstart=blk; first=0; } if(!sb->protocol2->bsighead) { sb->protocol2->bsighead=blk; } blist_add_blk(blist, blk); } else blk_free(&blk); blk=NULL; } if(blist->tail) sb->protocol2->bend=blist->tail; return 1; }
// Combine the phase1 and phase2 files into a new manifest. int backup_phase3_server_all(struct sdirs *sdirs, struct conf **confs) { int ret=-1; int pcmp=0; struct blk *blk=NULL; struct sbuf *usb=NULL; struct sbuf *csb=NULL; char *manifesttmp=NULL; struct manio *newmanio=NULL; struct manio *chmanio=NULL; struct manio *unmanio=NULL; enum protocol protocol=get_protocol(confs); struct cntr *cntr=get_cntr(confs); const char *rmanifest_relative=NULL; logp("Begin phase3 (merge manifests)\n"); if(protocol==PROTO_2) rmanifest_relative=get_rmanifest_relative(sdirs, confs); if(!(manifesttmp=get_tmp_filename(sdirs->manifest)) || !(newmanio=manio_open_phase3(manifesttmp, comp_level(get_int(confs[OPT_COMPRESSION])), protocol, rmanifest_relative)) || !(chmanio=manio_open_phase2(sdirs->changed, "rb", protocol)) || !(unmanio=manio_open_phase2(sdirs->unchanged, "rb", protocol)) || !(usb=sbuf_alloc(protocol)) || !(csb=sbuf_alloc(protocol))) goto end; while(chmanio || unmanio) { if(!blk && !(blk=blk_alloc())) goto end; if(unmanio && !usb->path.buf) { switch(manio_read(unmanio, usb)) { case -1: goto end; case 1: manio_close(&unmanio); } } if(chmanio && !csb->path.buf) { switch(manio_read(chmanio, csb)) { case -1: goto end; case 1: manio_close(&chmanio); } } if(usb->path.buf && !csb->path.buf) { if(write_status(CNTR_STATUS_MERGING, usb->path.buf, cntr)) goto end; switch(manio_copy_entry( usb, usb, &blk, unmanio, newmanio)) { case -1: goto end; case 1: manio_close(&unmanio); } } else if(!usb->path.buf && csb->path.buf) { if(write_status(CNTR_STATUS_MERGING, csb->path.buf, cntr)) goto end; switch(manio_copy_entry( csb, csb, &blk, chmanio, newmanio)) { case -1: goto end; case 1: manio_close(&chmanio); } } else if(!usb->path.buf && !csb->path.buf) { continue; } else if(!(pcmp=sbuf_pathcmp(usb, csb))) { // They were the same - write one. if(write_status(CNTR_STATUS_MERGING, csb->path.buf, cntr)) goto end; switch(manio_copy_entry( csb, csb, &blk, chmanio, newmanio)) { case -1: goto end; case 1: manio_close(&chmanio); } } else if(pcmp<0) { if(write_status(CNTR_STATUS_MERGING, usb->path.buf, cntr)) goto end; switch(manio_copy_entry( usb, usb, &blk, unmanio, newmanio)) { case -1: goto end; case 1: manio_close(&unmanio); } } else { if(write_status(CNTR_STATUS_MERGING, csb->path.buf, cntr)) goto end; switch(manio_copy_entry( csb, csb, &blk, chmanio, newmanio)) { case -1: goto end; case 1: manio_close(&chmanio); } } } // Flush to disk. if(manio_close(&newmanio)) { logp("error gzclosing %s in backup_phase3_server\n", manifesttmp); goto end; } // Rename race condition should be of no consequence here, as the // manifest should just get recreated automatically. if(do_rename(manifesttmp, sdirs->manifest)) goto end; else { recursive_delete(sdirs->changed); recursive_delete(sdirs->unchanged); } logp("End phase3 (merge manifests)\n"); ret=0; end: manio_close(&newmanio); manio_close(&chmanio); manio_close(&unmanio); sbuf_free(&csb); sbuf_free(&usb); blk_free(&blk); free_w(&manifesttmp); return ret; }
int main(int argc, char **argv) { CS_CONTEXT *ctx; CS_CONNECTION *conn; CS_COMMAND *cmd; CS_BLKDESC *blkdesc; int verbose = 0; int ret = 0; int i; char command[512]; static char table_name[20] = "hogexxx"; fprintf(stdout, "%s: Inserting data using bulk cancelling\n", __FILE__); if (verbose) { fprintf(stdout, "Trying login\n"); } ret = try_ctlogin(&ctx, &conn, &cmd, verbose); if (ret != CS_SUCCEED) { fprintf(stderr, "Login failed\n"); return 1; } sprintf(command, "if exists (select 1 from sysobjects where type = 'U' and name = '%s') drop table %s", table_name, table_name); ret = run_command(cmd, command); if (ret != CS_SUCCEED) return 1; ret = run_command(cmd, create_table_sql); if (ret != CS_SUCCEED) return 1; ret = blk_alloc(conn, BLK_VERSION_100, &blkdesc); if (ret != CS_SUCCEED) { fprintf(stderr, "blk_alloc() failed\n"); return 1; } for (i = 0; i < 10; i++) { /* compute some data */ memset(command, ' ', sizeof(command)); memset(command, 'a' + i, (i * 37) % 11); ret = hoge_blkin(conn, blkdesc, table_name, command); if (ret != CS_SUCCEED) return 1; } blk_drop(blkdesc); /* TODO test correct insert */ printf("done\n"); ret = try_ctlogout(ctx, conn, cmd, verbose); if (ret != CS_SUCCEED) { fprintf(stderr, "Logout failed\n"); return 1; } return 0; }
/** * @brief bulk copy data from TBL_ImsiMsisdn */ int CSybClient::bulkCopyOut_ImsiMsisdn(int connId, char* tablename) { CS_CONNECTION *connection = getConnection(connId); if(NULL == connection || NULL == tablename) return -1; CLEAR_ALL_MESSAGE(connection); int copiedRowCnt = 0; CS_RETCODE retcode; #define MAX_ROWS_ONETIME 1024 #define FIELD_IMSI 1 #define FIELD_MSISDN 2 CS_BLKDESC *blkdesc; CS_DATAFMT datafmt; /* hold the variable descriptions */ CS_CHAR msisdn_fieldbuf[MAX_ROWS_ONETIME][32]; /* point to the data for transfer */ CS_SMALLINT msisdn_fieldind[MAX_ROWS_ONETIME]; CS_INT msisdn_fieldlen[MAX_ROWS_ONETIME]; // CS_CHAR imsi_fieldbuf[MAX_ROWS_ONETIME][16]; CS_SMALLINT imsi_fieldind[MAX_ROWS_ONETIME]; CS_INT imsi_fieldlen[MAX_ROWS_ONETIME]; // CS_INT numrows = MAX_ROWS_ONETIME; /* ** Ready to start the bulk copy in now that all the connections ** have been made and have a table name. ** ** Start by getting the bulk descriptor and starting the initialization */ if ( (retcode = blk_alloc(connection, EX_BULK_VERSION, &blkdesc)) != CS_SUCCEED) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); return -1; } if (blk_init(blkdesc, CS_BLK_OUT, tablename, CS_NULLTERM) == CS_FAIL) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); blk_drop(blkdesc); return -1; } /* ** Now to bind the variables to the columns and transfer the data */ datafmt.datatype = CS_CHAR_TYPE; datafmt.format = CS_FMT_NULLTERM; datafmt.maxlength= 16; datafmt.count = MAX_ROWS_ONETIME; datafmt.locale = 0;// if (blk_bind(blkdesc, FIELD_IMSI, &datafmt, imsi_fieldbuf, imsi_fieldlen, imsi_fieldind) != CS_SUCCEED) // bind IMSI field { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); blk_drop(blkdesc); return -1; } // datafmt.maxlength= 32; // if (blk_bind(blkdesc, FIELD_MSISDN, &datafmt, msisdn_fieldbuf, msisdn_fieldlen, msisdn_fieldind) != CS_SUCCEED) // bind MSISDN field { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); blk_drop(blkdesc); return -1; } while( (retcode = blk_rowxfer_mult(blkdesc, &numrows)) == CS_SUCCEED || retcode == CS_ROW_FAIL) { copiedRowCnt += numrows; for(int ii = 0; ii < numrows; ++ii) { stIMSI imsi; stMSISDN msisdn; imsi.m_ucIMSILen = strlen(imsi_fieldbuf[ii]); memcpy(imsi.m_szIMSI, imsi_fieldbuf[ii], imsi.m_ucIMSILen); msisdn.ucLen = strlen(msisdn_fieldbuf[ii]); memcpy(msisdn.ucMSISDN, msisdn_fieldbuf[ii], msisdn.ucLen); m_pParentThread->addIMSI2MSISDN(msisdn, imsi); } } // if(retcode != CS_END_DATA) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); retcode = CS_FAIL; } /* ** ALL the rows sent so clear up */ CS_INT blktype = copiedRowCnt > 0 ? CS_BLK_ALL : CS_BLK_CANCEL; if ( (retcode = blk_done(blkdesc, blktype, &numrows)) == CS_FAIL) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); } if ( (retcode = blk_drop(blkdesc)) == CS_FAIL) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); } return retcode == CS_SUCCEED ? copiedRowCnt : -1; }
// Return -1 for error, 0 for entry not changed, 1 for entry changed (or new). static int entry_changed(struct asfd *asfd, struct sbuf *sb, struct manio *cmanio, struct manio *unmanio, struct conf *conf) { static int finished=0; static struct sbuf *csb=NULL; static struct blk *blk=NULL; if(finished) return 1; if(!csb && !(csb=sbuf_alloc(conf))) return -1; if(csb->path.buf) { // Already have an entry. } else { // Need to read another. if(!blk && !(blk=blk_alloc())) return -1; switch(manio_sbuf_fill(cmanio, asfd, csb, blk, NULL, conf)) { case 1: // Reached the end. sbuf_free(&csb); blk_free(&blk); finished=1; return 1; case -1: return -1; } if(!csb->path.buf) { logp("Should have a path at this point, but do not, in %s\n", __func__); return -1; } // Got an entry. } while(1) { switch(sbuf_pathcmp(csb, sb)) { case 0: return found_in_current_manifest(asfd, csb, sb, cmanio, unmanio, &blk, conf); case 1: return 1; case -1: // Behind - need to read more data from the old // manifest. switch(manio_sbuf_fill(cmanio, asfd, csb, blk, NULL, conf)) { case -1: return -1; case 1: { // Reached the end. sbuf_free(&csb); blk_free(&blk); return 1; } } // Got something, go back around the loop. } } return 0; }
/** * @brief bulk copy data from TBL_NumberPrefix */ int CSybClient::bulkCopyOut_NumberPrefix(int connId, char* tablename) { CS_CONNECTION *connection = getConnection(connId); if(NULL == connection || NULL == tablename) return -1; CLEAR_ALL_MESSAGE(connection); int copiedRowCnt = 0; CS_RETCODE retcode; #define MAX_ROWS_ONETIME 1024 CS_BLKDESC *blkdesc; CS_DATAFMT datafmt; /* hold the variable descriptions */ CS_CHAR areaCode_fieldBuf[MAX_ROWS_ONETIME][16]; CS_CHAR prefix_fieldBuf[MAX_ROWS_ONETIME][32]; /* point to the data for transfer */ CS_DATETIME updateTime_fieldBuf[MAX_ROWS_ONETIME]; CS_TINYINT sarNumber_fieldBuf[MAX_ROWS_ONETIME]; CS_TINYINT exchangeHouse_fieldBuf[MAX_ROWS_ONETIME]; // CS_SMALLINT areaCode_indBuf[MAX_ROWS_ONETIME]; CS_SMALLINT prefix_indBuf[MAX_ROWS_ONETIME]; CS_SMALLINT updateTime_indBuf[MAX_ROWS_ONETIME]; CS_SMALLINT sarNumber_indBuf[MAX_ROWS_ONETIME]; CS_SMALLINT exchangeHouse_indBuf[MAX_ROWS_ONETIME]; // CS_INT areaCode_bufLen[MAX_ROWS_ONETIME]; CS_INT prefix_bufLen[MAX_ROWS_ONETIME]; CS_INT updateTime_bufLen[MAX_ROWS_ONETIME]; CS_INT sarNumber_bufLen[MAX_ROWS_ONETIME]; CS_INT exchangeHouse_bufLen[MAX_ROWS_ONETIME]; // CS_INT numrows = MAX_ROWS_ONETIME; // #define FIELD_AreaCode 1 #define FIELD_Prefix 2 #define FIELD_UpdateTime 3 #define FIELD_SARNumber 5 #define FIELD_ExchangeHouse 6 /* ** Ready to start the bulk copy in now that all the connections ** have been made and have a table name. ** ** Start by getting the bulk descriptor and starting the initialization */ if ( (retcode = blk_alloc(connection, EX_BULK_VERSION, &blkdesc)) != CS_SUCCEED) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); return -1; } if (blk_init(blkdesc, CS_BLK_OUT, tablename, CS_NULLTERM) == CS_FAIL) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); blk_drop(blkdesc); return -1; } /* ** Now to bind the variables to the columns and transfer the data */ datafmt.datatype = CS_CHAR_TYPE; datafmt.format = CS_FMT_NULLTERM; datafmt.maxlength= 16; datafmt.count = MAX_ROWS_ONETIME; datafmt.locale = 0;// if (blk_bind(blkdesc, FIELD_AreaCode, &datafmt, areaCode_fieldBuf, areaCode_bufLen, areaCode_indBuf) != CS_SUCCEED) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); blk_drop(blkdesc); return -1; } // datafmt.datatype = CS_CHAR_TYPE; datafmt.format = CS_FMT_NULLTERM; datafmt.maxlength= 32; if (blk_bind(blkdesc, FIELD_Prefix, &datafmt, prefix_fieldBuf, prefix_bufLen, prefix_indBuf) != CS_SUCCEED) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); blk_drop(blkdesc); return -1; } // datafmt.datatype = CS_DATETIME_TYPE; datafmt.format = CS_FMT_UNUSED; datafmt.maxlength= CS_SIZEOF(CS_DATETIME); if (blk_bind(blkdesc, FIELD_UpdateTime, &datafmt, updateTime_fieldBuf, updateTime_bufLen, updateTime_indBuf) != CS_SUCCEED) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); blk_drop(blkdesc); return -1; } // datafmt.datatype = CS_TINYINT_TYPE; datafmt.format = CS_FMT_UNUSED; datafmt.maxlength= CS_SIZEOF(CS_TINYINT); if (blk_bind(blkdesc, FIELD_SARNumber, &datafmt, sarNumber_fieldBuf, sarNumber_bufLen, sarNumber_indBuf) != CS_SUCCEED) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); blk_drop(blkdesc); return -1; } // if (blk_bind(blkdesc, FIELD_ExchangeHouse, &datafmt, exchangeHouse_fieldBuf, exchangeHouse_bufLen, exchangeHouse_indBuf) != CS_SUCCEED) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); blk_drop(blkdesc); return -1; } //if (blk_rowxfer_mult(blkdesc, &numrows) == CS_FAIL) while( (retcode = blk_rowxfer_mult(blkdesc, &numrows)) == CS_SUCCEED || retcode == CS_ROW_FAIL) { copiedRowCnt += numrows; // for(int ii = 0; ii < numrows; ++ii) { // stNumPrefix prefix; prefix.prefixLen = strlen(prefix_fieldBuf[ii]); memcpy(prefix.szPrefix, prefix_fieldBuf[ii], prefix.prefixLen); prefix.areaCodeLen = strlen(areaCode_fieldBuf[ii]); memcpy(prefix.szAreaCode, areaCode_fieldBuf[ii], prefix.areaCodeLen); prefix.adminCenterId = sarNumber_fieldBuf[ii]; prefix.switchCenterId = exchangeHouse_fieldBuf[ii]; m_pParentThread->addNumPrefix(prefix); } } // if(retcode != CS_END_DATA) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); retcode = CS_FAIL; } /* ** ALL the rows sent so clear up */ CS_INT blktype = copiedRowCnt > 0 ? CS_BLK_ALL : CS_BLK_CANCEL; if ( (retcode = blk_done(blkdesc, blktype, &numrows)) == CS_FAIL) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); } if ( (retcode = blk_drop(blkdesc)) == CS_FAIL) { DIAG_CLIENT_MESSAGE(connection); DIAG_SERVER_MESSAGE(connection); } return retcode == CS_SUCCEED ? copiedRowCnt : -1; }