int blk_verify_fingerprint(uint64_t fingerprint, char *data, size_t length)
{
win_reset();
memcpy(gbuf, data, length);
gbuf_end=gbuf+length;
gcp=gbuf;
blk_free(&blk);
if(!blk && !(blk=blk_alloc())) return -1;
blk->length=0;
blk->fingerprint=0;
// FIX THIS: blk_read should return 1 when it has a block.
// But, if the block is too small (because the end of the file
// happened), it returns 0, and blks_generate treats it as having found
// a final block.
// So, here the return of blk_read is ignored and we look at the
// position of gcp instead.
blk_read();
//printf("%d %d\n", blk->length, length);
//printf("%016"PRIX64" %016"PRIX64" ",
// blk->fingerprint, fingerprint);
//printf("%s\n", blk->fingerprint==fingerprint?"yes":"no");
if(gcp==gbuf_end
&& blk->fingerprint==fingerprint)
return 1;
return 0;
}
// The server uses this for verification.
int blk_read_verify(struct blk *blk_to_verify)
{
if(!win)
{
rconf_init(&rconf);
if(!(win=win_alloc(&rconf))) return -1;
}
gbuf=blk_to_verify->data;
gbuf_end=gbuf+blk_to_verify->length;
gcp=gbuf;
if(!blk && !(blk=blk_alloc())) return -1;
blk->length=0;
blk->fingerprint=0;
// FIX THIS: blk_read should return 1 when it has a block.
// But, if the block is too small (because the end of the file
// happened), it returns 0, and blks_generate treats it as having found
// a final block.
// So, here the return of blk_read is ignored and we look at the
// position of gcp instead.
blk_read();
if(gcp==gbuf_end
&& blk->fingerprint==blk_to_verify->fingerprint)
return 1;
return 0;
}
// Return -1 for error, 0 for entry not changed, 1 for entry changed (or new).
static int entry_changed(struct sbuf *sb,
struct manios *manios, struct asfd *chfd, struct sbuf **csb)
{
static int finished=0;
static struct blk *blk=NULL;
if(finished) return 1;
if((*csb)->path.buf)
{
// Already have an entry.
}
else
{
// Need to read another.
if(!blk && !(blk=blk_alloc())) return -1;
switch(manio_read_with_blk(manios->current,
*csb, blk, NULL))
{
case 1: // Reached the end.
sbuf_free(csb);
blk_free(&blk);
finished=1;
return 1;
case -1: return -1;
}
if(!(*csb)->path.buf)
{
logp("Should have a path at this point, but do not, in %s\n", __func__);
return -1;
}
// Got an entry.
}
while(1)
{
switch(sbuf_pathcmp(*csb, sb))
{
case 0: return found_in_current_manifest(*csb, sb,
manios, &blk, chfd);
case 1: return 1;
case -1:
// Behind - need to read more data from the old
// manifest.
switch(manio_read_with_blk(manios->current,
*csb, blk, NULL))
{
case 1: // Reached the end.
sbuf_free(csb);
blk_free(&blk);
return 1;
case -1: return -1;
}
// Got something, go back around the loop.
}
}
return 0;
}
static void add_blk_and_data_files(struct blist *blist, uint64_t save_path)
{
struct blk *blk;
fail_unless((blk=blk_alloc())!=NULL);
blk->fingerprint=prng_next64();
prng_md5sum(blk->md5sum);
blk->savepath=save_path;
blist_add_blk(blist, blk);
}
static void add_blk(struct blist *blist)
{
struct blk *blk;
fail_unless((blk=blk_alloc())!=NULL);
blk->fingerprint=prng_next64();
prng_md5sum(blk->md5sum);
blk->savepath=prng_next64();
blist_add_blk(blist, blk);
}
static void* hashstr_alloc(void **b, int n, uint x) {
struct hashstrblk *m = 0; void *r[2] = { 0, 0 };
if(!b[0]) b[0] = blk_create(x);
if(n > 0) if(m = blk_alloc(b[0], sizeof(int) + n * sizeof(char*))) {
m->n = n - 1; memset(m->h, 0, n << 2); b[1] = m;
}
return (void*)m;
}
struct blk *blk_alloc_with_data(uint32_t max_data_length)
{
struct blk *blk=NULL;
if(!(blk=blk_alloc())) return NULL;
if((blk->data=(char *)
calloc_w(1, sizeof(char)*max_data_length, __func__)))
return blk;
blk_free(&blk);
return NULL;
}
// When a backup is ongoing, use this to add newly complete candidates.
int candidate_add_fresh(const char *path, struct conf *conf)
{
int ars;
int ret=-1;
gzFile zp=NULL;
const char *cp=NULL;
struct sbuf *sb=NULL;
struct candidate *candidate=NULL;
struct blk *blk=NULL;
if(!(candidate=candidates_add_new())) goto end;
cp=path+strlen(conf->directory);
while(cp && *cp=='/') cp++;
if(!(candidate->path=strdup(cp)))
{
log_out_of_memory(__func__);
goto end;
}
if(!(sb=sbuf_alloc(conf))
|| !(blk=blk_alloc())
|| !(zp=gzopen_file(path, "rb")))
goto end;
while(zp)
{
if((ars=sbuf_fill_from_gzfile(sb, NULL /* struct async */,
zp, blk, NULL, conf))<0)
goto end;
else if(ars>0)
{
// Reached the end.
break;
}
if(!*(blk->weak)) continue;
if(is_hook(blk->weak))
{
if(sparse_add_candidate(blk->weak, candidate))
goto end;
}
*blk->weak='\0';
}
if(scores_grow(scores, candidates_len)) goto end;
candidates_set_score_pointers(candidates, candidates_len, scores);
scores_reset(scores);
//printf("HERE: %d candidates\n", (int)candidates_len);
ret=0;
end:
gzclose_fp(&zp);
sbuf_free(sb);
blk_free(blk);
return ret;
}
static int write_to_dpth(struct dpth *dpth, const char *savepathstr)
{
int ret;
struct iobuf wbuf;
struct blk *blk=blk_alloc();
savepathstr_to_bytes(savepathstr, blk->savepath);
wbuf.buf=strdup_w("abc", __FUNCTION__);
wbuf.len=3;
ret=dpth_protocol2_fwrite(dpth, &wbuf, blk);
free_w(&wbuf.buf);
blk_free(&blk);
return ret;
}
void hash_add_elem_idx(Hash* hash, const Elem* elem, int idx)
{
HElem* he = blk_alloc(sizeof(*he));
unsigned int hcode;
assert(hash_is_valid(hash));
assert(elem_is_valid(elem));
assert(he != NULL);
hcode = elem_hash(elem) % hash->size;
he->value.elem_idx.elem = elem;
he->value.elem_idx.idx = idx;
he->next = hash->bucket[hcode];
hash->bucket[hcode] = he;
hash->elems++;
}
void hash_add_numb(Hash* hash, const Numb* numb)
{
HElem* he = blk_alloc(sizeof(*he));
unsigned int hcode;
assert(hash_is_valid(hash));
assert(numb_is_valid(numb));
assert(hash->type == HASH_NUMB);
assert(he != NULL);
hcode = numb_hash(numb) % hash->size;
he->value.numb = numb;
he->next = hash->bucket[hcode];
hash->bucket[hcode] = he;
hash->elems++;
}
void hash_add_tuple(Hash* hash, const Tuple* tuple)
{
HElem* he = blk_alloc(sizeof(*he));
unsigned int hcode;
assert(hash_is_valid(hash));
assert(tuple_is_valid(tuple));
assert(hash->type == HASH_TUPLE);
assert(he != NULL);
hcode = tuple_hash(tuple) % hash->size;
he->value.tuple = tuple;
he->next = hash->bucket[hcode];
hash->bucket[hcode] = he;
hash->elems++;
}
void hash_add_mono(Hash* hash, const Mono* mono)
{
HElem* he = blk_alloc(sizeof(*he));
unsigned int hcode;
assert(hash_is_valid(hash));
assert(mono_is_valid(mono));
assert(hash->type == HASH_MONO);
assert(he != NULL);
hcode = mono_hash(mono) % hash->size;
he->value.mono = mono;
he->next = hash->bucket[hcode];
hash->bucket[hcode] = he;
hash->elems++;
}
static void list_add_data(List* list, const ListData* data)
{
ListElem* elem = blk_alloc(sizeof(*elem));
assert(list_is_valid(list));
assert(elem != NULL);
assert(data != NULL);
elem->data = *data;
elem->next = &list->anchor;
elem->prev = list->anchor.prev;
list->anchor.prev->next = elem;
list->anchor.prev = elem;
list->elems++;
}
void hash_add_entry(Hash* hash, const Entry* entry)
{
HElem* he = blk_alloc(sizeof(*he));
const Tuple* tuple;
unsigned int hcode;
assert(hash_is_valid(hash));
assert(entry_is_valid(entry));
assert(hash->type == HASH_ENTRY);
assert(he != NULL);
tuple = entry_get_tuple(entry);
hcode = tuple_hash(tuple) % hash->size;
he->value.entry = entry;
he->next = hash->bucket[hcode];
hash->bucket[hcode] = he;
hash->elems++;
}
static
#endif
int champ_server_deal_with_rbuf_sig(struct asfd *asfd,
const char *directory, struct scores *scores)
{
struct blk *blk;
if(!(blk=blk_alloc())) return -1;
blist_add_blk(asfd->blist, blk);
if(!asfd->blist->blk_to_dedup)
asfd->blist->blk_to_dedup=blk;
if(blk_set_from_iobuf_sig(blk, asfd->rbuf))
return -1;
//logp("Got fingerprint from %d: %lu - %lu\n",
// asfd->fd, blk->index, blk->fingerprint);
return deduplicate_maybe(asfd, blk, directory, scores);
}
static void
wb_init(struct write_block *wb , struct block *b) {
if (b==NULL) {
wb->head = blk_alloc();
wb->len = 0;
wb->current = wb->head;
wb->ptr = 0;
wb_push(wb, &wb->len, sizeof(wb->len));
} else {
wb->head = b;
int * plen = (int *)b->buffer;
int sz = *plen;
wb->len = sz;
while (b->next) {
sz -= BLOCK_SIZE;
b = b->next;
}
wb->current = b;
wb->ptr = sz;
}
}
inline static void
wb_push(struct write_block *b, const void *buf, int sz) {
const char * buffer = buf;
if (b->ptr == BLOCK_SIZE) {
_again:
b->current = b->current->next = blk_alloc();
b->ptr = 0;
}
if (b->ptr <= BLOCK_SIZE - sz) {
memcpy(b->current->buffer + b->ptr, buffer, sz);
b->ptr+=sz;
b->len+=sz;
} else {
int copy = BLOCK_SIZE - b->ptr;
memcpy(b->current->buffer + b->ptr, buffer, copy);
buffer += copy;
b->len += copy;
sz -= copy;
goto _again;
}
}
static uint hashstr_add(void **b, uint p, uchar *s, uint n) {
uint a, i = 0, h = 0; struct hashstrblk *x; struct hashstritem *m;
if(!b || !(x = b[1])) return p;
for(;;) {
if(i >= n) break;
a = s[i]; if(n == -1 && !a) break;
i++; h = h * 25 + a;
}
n = h & x->n;
m = x->h[n];
while(m) {
if(m->h == h && m->i == i) if(!memcmp(s, m->s, i)) return m->p;
m = m->n;
}
if(m = blk_alloc(b[0], sizeof(void*) + 3 * sizeof(int) + i * sizeof(char))) {
m->h = h; m->i = i; m->p = p;
memcpy(m->s, s, i);
m->n = x->h[n]; x->h[n] = m;
}
return p;
}
static int add_to_sig_list(struct slist *slist, struct iobuf *rbuf)
{
// Goes on slist->add_sigs_here
struct blk *blk;
struct protocol2 *protocol2;
if(!(blk=blk_alloc())) return -1;
blist_add_blk(slist->blist, blk);
protocol2=slist->add_sigs_here->protocol2;
if(!protocol2->bstart) protocol2->bstart=blk;
if(!protocol2->bsighead) protocol2->bsighead=blk;
if(blk_set_from_iobuf_sig(blk, rbuf)) return -1;
// Need to send sigs to champ chooser, therefore need to point
// to the oldest unsent one if nothing is pointed to yet.
if(!slist->blist->blk_for_champ_chooser)
slist->blist->blk_for_champ_chooser=blk;
return 0;
}
static int add_to_sig_list(struct slist *slist, struct blist *blist,
struct iobuf *rbuf, struct dpth *dpth, struct conf *conf)
{
// Goes on slist->add_sigs_here
struct blk *blk;
struct burp2 *burp2;
if(!(blk=blk_alloc())) return -1;
blist_add_blk(blist, blk);
burp2=slist->add_sigs_here->burp2;
if(!burp2->bstart) burp2->bstart=blk;
if(!burp2->bsighead) burp2->bsighead=blk;
if(split_sig(rbuf->buf, rbuf->len,
&blk->fingerprint, blk->md5sum)) return -1;
// Need to send sigs to champ chooser, therefore need to point
// to the oldest unsent one if nothing is pointed to yet.
if(!blist->blk_for_champ_chooser) blist->blk_for_champ_chooser=blk;
return 0;
}
// This is basically backup_phase3_server() from protocol1. It used to merge the
// unchanged and changed data into a single file. Now it splits the manifests
// into several files.
int backup_phase3_server_protocol2(struct sdirs *sdirs, struct conf **confs)
{
int ret=1;
int pcmp=0;
char *hooksdir=NULL;
char *dindexdir=NULL;
char *manifesttmp=NULL;
struct sbuf *usb=NULL;
struct sbuf *csb=NULL;
struct blk *blk=NULL;
int finished_ch=0;
int finished_un=0;
struct manio *newmanio=NULL;
struct manio *chmanio=NULL;
struct manio *unmanio=NULL;
uint64_t fcount=0;
logp("Start phase3\n");
if(!(manifesttmp=get_tmp_filename(sdirs->rmanifest))
|| !(newmanio=manio_alloc())
|| !(chmanio=manio_alloc())
|| !(unmanio=manio_alloc())
|| !(hooksdir=prepend_s(manifesttmp, "hooks"))
|| !(dindexdir=prepend_s(manifesttmp, "dindex"))
|| manio_init_write(newmanio, manifesttmp)
|| manio_init_write_hooks(newmanio,
get_string(confs[OPT_DIRECTORY]), hooksdir, sdirs->rmanifest)
|| manio_init_write_dindex(newmanio, dindexdir)
|| manio_init_read(chmanio, sdirs->changed)
|| manio_init_read(unmanio, sdirs->unchanged)
|| !(usb=sbuf_alloc(confs))
|| !(csb=sbuf_alloc(confs)))
goto end;
while(!finished_ch || !finished_un)
{
if(!blk && !(blk=blk_alloc())) goto end;
if(!finished_un
&& usb
&& !usb->path.buf)
{
switch(manio_sbuf_fill(unmanio, NULL /* no async */,
usb, NULL, NULL, confs))
{
case -1: goto end;
case 1: finished_un++;
}
}
if(!finished_ch
&& csb
&& !csb->path.buf)
{
switch(manio_sbuf_fill(chmanio, NULL /* no async */,
csb, NULL, NULL, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
if((usb && usb->path.buf) && (!csb || !csb->path.buf))
{
switch(manio_copy_entry(NULL /* no async */,
&usb, usb,
&blk, unmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_un++;
}
}
else if((!usb || !usb->path.buf) && (csb && csb->path.buf))
{
switch(manio_copy_entry(NULL /* no async */,
&csb, csb, &blk, chmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
else if((!usb || !usb->path.buf) && (!csb || !(csb->path.buf)))
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(usb, csb)))
{
// They were the same - write one.
switch(manio_copy_entry(NULL /* no async */,
&csb, csb, &blk, chmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
else if(pcmp<0)
{
switch(manio_copy_entry(NULL /* no async */,
&usb, usb, &blk, unmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_un++;
}
}
else
{
switch(manio_copy_entry(NULL /* no async */,
&csb, csb, &blk, chmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
}
fcount=newmanio->fcount;
// Flush to disk and set up for reading.
if(manio_free(&newmanio)
|| !(newmanio=manio_alloc())
|| manio_init_read(newmanio, sdirs->rmanifest))
goto end;
// Rename race condition should be of no consequence here, as the
// manifest should just get recreated automatically.
if(do_rename(manifesttmp, sdirs->rmanifest))
goto end;
else
{
recursive_delete(sdirs->changed, NULL, 1);
recursive_delete(sdirs->unchanged, NULL, 1);
}
if(sparse_generation(newmanio, fcount, sdirs, confs))
goto end;
ret=0;
logp("End phase3\n");
end:
manio_free(&newmanio);
manio_free(&chmanio);
manio_free(&unmanio);
sbuf_free(&csb);
sbuf_free(&usb);
blk_free(&blk);
free_w(&hooksdir);
free_w(&dindexdir);
free_w(&manifesttmp);
return ret;
}
// This deals with reading in the sparse index, as well as actual candidate
// manifests.
enum cand_ret candidate_load(struct candidate *candidate, const char *path,
struct scores *scores)
{
enum cand_ret ret=CAND_RET_PERM;
struct fzp *fzp=NULL;
struct sbuf *sb=NULL;
struct blk *blk=NULL;
if(!(sb=sbuf_alloc(PROTO_2))
|| !(blk=blk_alloc()))
{
ret=CAND_RET_PERM;
goto error;
}
if(!(fzp=fzp_gzopen(path, "rb")))
{
ret=CAND_RET_TEMP;
goto error;
}
while(fzp)
{
sbuf_free_content(sb);
switch(sbuf_fill_from_file(sb, fzp, blk, NULL))
{
case 1: goto end;
case -1:
logp("Error reading %s in %s, pos %d\n",
path, __func__, fzp_tell(fzp));
ret=CAND_RET_TEMP;
goto error;
}
if(blk_fingerprint_is_hook(blk))
{
if(sparse_add_candidate(&blk->fingerprint, candidate))
{
ret=CAND_RET_PERM;
goto error;
}
}
else if(sb->path.cmd==CMD_MANIFEST)
{
if(!(candidate=candidates_add_new()))
{
ret=CAND_RET_PERM;
goto error;
}
candidate->path=sb->path.buf;
sb->path.buf=NULL;
}
blk->fingerprint=0;
}
end:
if(scores_grow(scores, candidates_len))
{
ret=CAND_RET_PERM;
goto error;
}
candidates_set_score_pointers(candidates, candidates_len, scores);
scores_reset(scores);
//logp("Now have %d candidates\n", (int)candidates_len);
ret=CAND_RET_OK;
error:
fzp_close(&fzp);
sbuf_free(&sb);
blk_free(&blk);
return ret;
}
int do_restore_client(struct asfd *asfd,
struct conf **confs, enum action act, int vss_restore)
{
int ret=-1;
char msg[512]="";
struct sbuf *sb=NULL;
struct blk *blk=NULL;
BFILE *bfd=NULL;
char *fullpath=NULL;
char *style=NULL;
char *datpath=NULL;
struct cntr *cntr=get_cntr(confs);
enum protocol protocol=get_protocol(confs);
int strip=get_int(confs[OPT_STRIP]);
int overwrite=get_int(confs[OPT_OVERWRITE]);
const char *backup=get_string(confs[OPT_BACKUP]);
const char *regex=get_string(confs[OPT_REGEX]);
const char *restore_prefix=get_string(confs[OPT_RESTOREPREFIX]);
const char *encryption_password=get_string(confs[OPT_ENCRYPTION_PASSWORD]);
if(!(bfd=bfile_alloc())) goto end;
bfile_init(bfd, 0, cntr);
snprintf(msg, sizeof(msg), "%s %s:%s", act_str(act),
backup?backup:"", regex?regex:"");
logp("doing %s\n", msg);
if(asfd->write_str(asfd, CMD_GEN, msg)
|| asfd_read_expect(asfd, CMD_GEN, "ok"))
goto error;
logp("doing %s confirmed\n", act_str(act));
#if defined(HAVE_WIN32)
if(act==ACTION_RESTORE) win32_enable_backup_privileges();
#endif
if(!(style=get_restore_style(asfd, confs)))
goto error;
if(!strcmp(style, RESTORE_SPOOL))
{
if(restore_spool(asfd, confs, &datpath))
goto error;
}
else
logp("Streaming restore direct\n");
logf("\n");
if(get_int(confs[OPT_SEND_CLIENT_CNTR]) && cntr_recv(asfd, confs))
goto error;
if(!(sb=sbuf_alloc(protocol))
|| (protocol==PROTO_2 && !(blk=blk_alloc())))
{
log_and_send_oom(asfd, __func__);
goto error;
}
while(1)
{
sbuf_free_content(sb);
if(protocol==PROTO_1)
sb->flags |= SBUF_CLIENT_RESTORE_HACK;
switch(sbuf_fill_from_net(sb, asfd, blk, datpath, cntr))
{
case 0: break;
case 1: if(asfd->write_str(asfd, CMD_GEN,
"restoreend ok")) goto error;
goto end; // It was OK.
default:
case -1: goto error;
}
if(protocol==PROTO_2)
{
if(blk->data)
{
int wret=0;
if(act==ACTION_VERIFY)
cntr_add(cntr, CMD_DATA, 1);
else
wret=write_data(asfd, bfd, blk);
if(!datpath) blk_free_content(blk);
blk->data=NULL;
if(wret) goto error;
continue;
}
else if(sb->endfile.buf)
{
continue;
}
}
switch(sb->path.cmd)
{
case CMD_DIRECTORY:
case CMD_FILE:
case CMD_ENC_FILE:
case CMD_SOFT_LINK:
case CMD_HARD_LINK:
case CMD_SPECIAL:
case CMD_METADATA:
case CMD_ENC_METADATA:
case CMD_VSS:
case CMD_ENC_VSS:
case CMD_VSS_T:
case CMD_ENC_VSS_T:
case CMD_EFS_FILE:
if(strip)
{
int s;
s=strip_path_components(asfd,
sb, strip, cntr, protocol);
if(s<0) goto error;
if(s==0)
{
// Too many components stripped
// - carry on.
continue;
}
// It is OK, sb.path is now stripped.
}
free_w(&fullpath);
if(!(fullpath=prepend_s(restore_prefix,
sb->path.buf)))
{
log_and_send_oom(asfd, __func__);
goto error;
}
if(act==ACTION_RESTORE)
{
strip_invalid_characters(&fullpath);
if(!overwrite_ok(sb, overwrite,
#ifdef HAVE_WIN32
bfd,
#endif
fullpath))
{
char msg[512]="";
// Something exists at that path.
snprintf(msg, sizeof(msg),
"Path exists: %s\n", fullpath);
if(restore_interrupt(asfd,
sb, msg, cntr, protocol))
goto error;
continue;
}
}
break;
case CMD_MESSAGE:
case CMD_WARNING:
log_recvd(&sb->path, cntr, 1);
logf("\n");
continue;
default:
break;
}
switch(sb->path.cmd)
{
// These are the same in both protocol1 and protocol2.
case CMD_DIRECTORY:
if(restore_dir(asfd, sb, fullpath, act, cntr,
protocol))
goto error;
continue;
case CMD_SOFT_LINK:
case CMD_HARD_LINK:
if(restore_link(asfd, sb, fullpath, act, cntr,
protocol, restore_prefix))
goto error;
continue;
case CMD_SPECIAL:
if(restore_special(asfd, sb,
fullpath, act, cntr, protocol))
goto error;
continue;
default:
break;
}
if(protocol==PROTO_2)
{
if(restore_switch_protocol2(asfd, sb, fullpath, act,
bfd, vss_restore, cntr))
goto error;
}
else
{
if(restore_switch_protocol1(asfd, sb, fullpath, act,
bfd, vss_restore, cntr, encryption_password))
goto error;
}
}
end:
ret=0;
error:
// It is possible for a fd to still be open.
bfd->close(bfd, asfd);
bfile_free(&bfd);
cntr_print_end(cntr);
cntr_print(cntr, act);
if(!ret) logp("%s finished\n", act_str(act));
else logp("ret: %d\n", ret);
sbuf_free(&sb);
free_w(&style);
if(datpath)
{
recursive_delete(datpath);
free_w(&datpath);
}
free_w(&fullpath);
blk_free(&blk);
return ret;
}
// The client uses this.
// Return 0 for OK. 1 for OK, and file ended, -1 for error.
int blks_generate(struct asfd *asfd, struct conf **confs,
struct sbuf *sb, struct blist *blist, int just_opened)
{
static ssize_t bytes;
first=just_opened;
if(!blk && !(blk=blk_alloc_with_data(rconf.blk_max)))
return -1;
if(gcp<gbuf_end)
{
// Could have got a fill before buf ran out -
// need to resume from the same place in that case.
if(blk_read_to_list(sb, blist))
return 0; // Got a block.
// Did not get a block. Carry on and read more.
}
while((bytes=rabin_read(sb, gbuf, rconf.blk_max)))
{
gcp=gbuf;
gbuf_end=gbuf+bytes;
sb->protocol2->bytes_read+=bytes;
if(blk_read_to_list(sb, blist))
return 0; // Got a block
// Did not get a block. Maybe should try again?
// If there are async timeouts, look at this!
return 0;
}
// Getting here means there is no more to read from the file.
// Make sure to deal with anything left over.
if(!sb->protocol2->bytes_read)
{
// Empty file, set up an empty block so that the server
// can skip over it.
if(!(sb->protocol2->bstart=blk_alloc())) return -1;
sb->protocol2->bsighead=blk;
blist_add_blk(blist, blk);
blk=NULL;
}
else if(blk)
{
if(blk->length)
{
if(first)
{
sb->protocol2->bstart=blk;
first=0;
}
if(!sb->protocol2->bsighead)
{
sb->protocol2->bsighead=blk;
}
blist_add_blk(blist, blk);
}
else blk_free(&blk);
blk=NULL;
}
if(blist->tail) sb->protocol2->bend=blist->tail;
return 1;
}
// Combine the phase1 and phase2 files into a new manifest.
int backup_phase3_server_all(struct sdirs *sdirs, struct conf **confs)
{
int ret=-1;
int pcmp=0;
struct blk *blk=NULL;
struct sbuf *usb=NULL;
struct sbuf *csb=NULL;
char *manifesttmp=NULL;
struct manio *newmanio=NULL;
struct manio *chmanio=NULL;
struct manio *unmanio=NULL;
enum protocol protocol=get_protocol(confs);
struct cntr *cntr=get_cntr(confs);
const char *rmanifest_relative=NULL;
logp("Begin phase3 (merge manifests)\n");
if(protocol==PROTO_2)
rmanifest_relative=get_rmanifest_relative(sdirs, confs);
if(!(manifesttmp=get_tmp_filename(sdirs->manifest))
|| !(newmanio=manio_open_phase3(manifesttmp,
comp_level(get_int(confs[OPT_COMPRESSION])),
protocol, rmanifest_relative))
|| !(chmanio=manio_open_phase2(sdirs->changed, "rb", protocol))
|| !(unmanio=manio_open_phase2(sdirs->unchanged, "rb", protocol))
|| !(usb=sbuf_alloc(protocol))
|| !(csb=sbuf_alloc(protocol)))
goto end;
while(chmanio || unmanio)
{
if(!blk && !(blk=blk_alloc())) goto end;
if(unmanio
&& !usb->path.buf)
{
switch(manio_read(unmanio, usb))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
if(chmanio
&& !csb->path.buf)
{
switch(manio_read(chmanio, csb))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
if(usb->path.buf && !csb->path.buf)
{
if(write_status(CNTR_STATUS_MERGING,
usb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
usb, usb, &blk, unmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
else if(!usb->path.buf && csb->path.buf)
{
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
else if(!usb->path.buf && !csb->path.buf)
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(usb, csb)))
{
// They were the same - write one.
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
else if(pcmp<0)
{
if(write_status(CNTR_STATUS_MERGING,
usb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
usb, usb, &blk, unmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
else
{
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
}
// Flush to disk.
if(manio_close(&newmanio))
{
logp("error gzclosing %s in backup_phase3_server\n",
manifesttmp);
goto end;
}
// Rename race condition should be of no consequence here, as the
// manifest should just get recreated automatically.
if(do_rename(manifesttmp, sdirs->manifest))
goto end;
else
{
recursive_delete(sdirs->changed);
recursive_delete(sdirs->unchanged);
}
logp("End phase3 (merge manifests)\n");
ret=0;
end:
manio_close(&newmanio);
manio_close(&chmanio);
manio_close(&unmanio);
sbuf_free(&csb);
sbuf_free(&usb);
blk_free(&blk);
free_w(&manifesttmp);
return ret;
}
int
main(int argc, char **argv)
{
CS_CONTEXT *ctx;
CS_CONNECTION *conn;
CS_COMMAND *cmd;
CS_BLKDESC *blkdesc;
int verbose = 0;
int ret = 0;
int i;
char command[512];
static char table_name[20] = "hogexxx";
fprintf(stdout, "%s: Inserting data using bulk cancelling\n", __FILE__);
if (verbose) {
fprintf(stdout, "Trying login\n");
}
ret = try_ctlogin(&ctx, &conn, &cmd, verbose);
if (ret != CS_SUCCEED) {
fprintf(stderr, "Login failed\n");
return 1;
}
sprintf(command, "if exists (select 1 from sysobjects where type = 'U' and name = '%s') drop table %s",
table_name, table_name);
ret = run_command(cmd, command);
if (ret != CS_SUCCEED)
return 1;
ret = run_command(cmd, create_table_sql);
if (ret != CS_SUCCEED)
return 1;
ret = blk_alloc(conn, BLK_VERSION_100, &blkdesc);
if (ret != CS_SUCCEED) {
fprintf(stderr, "blk_alloc() failed\n");
return 1;
}
for (i = 0; i < 10; i++) {
/* compute some data */
memset(command, ' ', sizeof(command));
memset(command, 'a' + i, (i * 37) % 11);
ret = hoge_blkin(conn, blkdesc, table_name, command);
if (ret != CS_SUCCEED)
return 1;
}
blk_drop(blkdesc);
/* TODO test correct insert */
printf("done\n");
ret = try_ctlogout(ctx, conn, cmd, verbose);
if (ret != CS_SUCCEED) {
fprintf(stderr, "Logout failed\n");
return 1;
}
return 0;
}
/**
* @brief bulk copy data from TBL_ImsiMsisdn
*/
int CSybClient::bulkCopyOut_ImsiMsisdn(int connId, char* tablename)
{
CS_CONNECTION *connection = getConnection(connId);
if(NULL == connection || NULL == tablename) return -1;
CLEAR_ALL_MESSAGE(connection);
int copiedRowCnt = 0;
CS_RETCODE retcode;
#define MAX_ROWS_ONETIME 1024
#define FIELD_IMSI 1
#define FIELD_MSISDN 2
CS_BLKDESC *blkdesc;
CS_DATAFMT datafmt; /* hold the variable descriptions */
CS_CHAR msisdn_fieldbuf[MAX_ROWS_ONETIME][32]; /* point to the data for transfer */
CS_SMALLINT msisdn_fieldind[MAX_ROWS_ONETIME];
CS_INT msisdn_fieldlen[MAX_ROWS_ONETIME];
//
CS_CHAR imsi_fieldbuf[MAX_ROWS_ONETIME][16];
CS_SMALLINT imsi_fieldind[MAX_ROWS_ONETIME];
CS_INT imsi_fieldlen[MAX_ROWS_ONETIME];
//
CS_INT numrows = MAX_ROWS_ONETIME;
/*
** Ready to start the bulk copy in now that all the connections
** have been made and have a table name.
**
** Start by getting the bulk descriptor and starting the initialization
*/
if ( (retcode = blk_alloc(connection, EX_BULK_VERSION, &blkdesc)) != CS_SUCCEED)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
return -1;
}
if (blk_init(blkdesc, CS_BLK_OUT, tablename, CS_NULLTERM) == CS_FAIL)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
blk_drop(blkdesc);
return -1;
}
/*
** Now to bind the variables to the columns and transfer the data
*/
datafmt.datatype = CS_CHAR_TYPE;
datafmt.format = CS_FMT_NULLTERM;
datafmt.maxlength= 16;
datafmt.count = MAX_ROWS_ONETIME;
datafmt.locale = 0;//
if (blk_bind(blkdesc, FIELD_IMSI, &datafmt, imsi_fieldbuf, imsi_fieldlen, imsi_fieldind) != CS_SUCCEED) // bind IMSI field
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
blk_drop(blkdesc);
return -1;
}
//
datafmt.maxlength= 32;
//
if (blk_bind(blkdesc, FIELD_MSISDN, &datafmt, msisdn_fieldbuf, msisdn_fieldlen, msisdn_fieldind) != CS_SUCCEED) // bind MSISDN field
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
blk_drop(blkdesc);
return -1;
}
while( (retcode = blk_rowxfer_mult(blkdesc, &numrows)) == CS_SUCCEED ||
retcode == CS_ROW_FAIL)
{
copiedRowCnt += numrows;
for(int ii = 0; ii < numrows; ++ii)
{
stIMSI imsi;
stMSISDN msisdn;
imsi.m_ucIMSILen = strlen(imsi_fieldbuf[ii]);
memcpy(imsi.m_szIMSI, imsi_fieldbuf[ii], imsi.m_ucIMSILen);
msisdn.ucLen = strlen(msisdn_fieldbuf[ii]);
memcpy(msisdn.ucMSISDN, msisdn_fieldbuf[ii], msisdn.ucLen);
m_pParentThread->addIMSI2MSISDN(msisdn, imsi);
}
}
//
if(retcode != CS_END_DATA)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
retcode = CS_FAIL;
}
/*
** ALL the rows sent so clear up
*/
CS_INT blktype = copiedRowCnt > 0 ? CS_BLK_ALL : CS_BLK_CANCEL;
if ( (retcode = blk_done(blkdesc, blktype, &numrows)) == CS_FAIL)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
}
if ( (retcode = blk_drop(blkdesc)) == CS_FAIL)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
}
return retcode == CS_SUCCEED ? copiedRowCnt : -1;
}
// Return -1 for error, 0 for entry not changed, 1 for entry changed (or new).
static int entry_changed(struct asfd *asfd, struct sbuf *sb,
struct manio *cmanio, struct manio *unmanio, struct conf *conf)
{
static int finished=0;
static struct sbuf *csb=NULL;
static struct blk *blk=NULL;
if(finished) return 1;
if(!csb && !(csb=sbuf_alloc(conf))) return -1;
if(csb->path.buf)
{
// Already have an entry.
}
else
{
// Need to read another.
if(!blk && !(blk=blk_alloc())) return -1;
switch(manio_sbuf_fill(cmanio, asfd, csb, blk, NULL, conf))
{
case 1: // Reached the end.
sbuf_free(&csb);
blk_free(&blk);
finished=1;
return 1;
case -1: return -1;
}
if(!csb->path.buf)
{
logp("Should have a path at this point, but do not, in %s\n", __func__);
return -1;
}
// Got an entry.
}
while(1)
{
switch(sbuf_pathcmp(csb, sb))
{
case 0: return found_in_current_manifest(asfd, csb, sb,
cmanio, unmanio, &blk, conf);
case 1: return 1;
case -1:
// Behind - need to read more data from the old
// manifest.
switch(manio_sbuf_fill(cmanio, asfd,
csb, blk, NULL, conf))
{
case -1: return -1;
case 1:
{
// Reached the end.
sbuf_free(&csb);
blk_free(&blk);
return 1;
}
}
// Got something, go back around the loop.
}
}
return 0;
}
/**
* @brief bulk copy data from TBL_NumberPrefix
*/
int CSybClient::bulkCopyOut_NumberPrefix(int connId, char* tablename)
{
CS_CONNECTION *connection = getConnection(connId);
if(NULL == connection || NULL == tablename) return -1;
CLEAR_ALL_MESSAGE(connection);
int copiedRowCnt = 0;
CS_RETCODE retcode;
#define MAX_ROWS_ONETIME 1024
CS_BLKDESC *blkdesc;
CS_DATAFMT datafmt; /* hold the variable descriptions */
CS_CHAR areaCode_fieldBuf[MAX_ROWS_ONETIME][16];
CS_CHAR prefix_fieldBuf[MAX_ROWS_ONETIME][32]; /* point to the data for transfer */
CS_DATETIME updateTime_fieldBuf[MAX_ROWS_ONETIME];
CS_TINYINT sarNumber_fieldBuf[MAX_ROWS_ONETIME];
CS_TINYINT exchangeHouse_fieldBuf[MAX_ROWS_ONETIME];
//
CS_SMALLINT areaCode_indBuf[MAX_ROWS_ONETIME];
CS_SMALLINT prefix_indBuf[MAX_ROWS_ONETIME];
CS_SMALLINT updateTime_indBuf[MAX_ROWS_ONETIME];
CS_SMALLINT sarNumber_indBuf[MAX_ROWS_ONETIME];
CS_SMALLINT exchangeHouse_indBuf[MAX_ROWS_ONETIME];
//
CS_INT areaCode_bufLen[MAX_ROWS_ONETIME];
CS_INT prefix_bufLen[MAX_ROWS_ONETIME];
CS_INT updateTime_bufLen[MAX_ROWS_ONETIME];
CS_INT sarNumber_bufLen[MAX_ROWS_ONETIME];
CS_INT exchangeHouse_bufLen[MAX_ROWS_ONETIME];
//
CS_INT numrows = MAX_ROWS_ONETIME;
//
#define FIELD_AreaCode 1
#define FIELD_Prefix 2
#define FIELD_UpdateTime 3
#define FIELD_SARNumber 5
#define FIELD_ExchangeHouse 6
/*
** Ready to start the bulk copy in now that all the connections
** have been made and have a table name.
**
** Start by getting the bulk descriptor and starting the initialization
*/
if ( (retcode = blk_alloc(connection, EX_BULK_VERSION, &blkdesc)) != CS_SUCCEED)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
return -1;
}
if (blk_init(blkdesc, CS_BLK_OUT, tablename, CS_NULLTERM) == CS_FAIL)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
blk_drop(blkdesc);
return -1;
}
/*
** Now to bind the variables to the columns and transfer the data
*/
datafmt.datatype = CS_CHAR_TYPE;
datafmt.format = CS_FMT_NULLTERM;
datafmt.maxlength= 16;
datafmt.count = MAX_ROWS_ONETIME;
datafmt.locale = 0;//
if (blk_bind(blkdesc, FIELD_AreaCode, &datafmt, areaCode_fieldBuf, areaCode_bufLen, areaCode_indBuf) != CS_SUCCEED)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
blk_drop(blkdesc);
return -1;
}
//
datafmt.datatype = CS_CHAR_TYPE;
datafmt.format = CS_FMT_NULLTERM;
datafmt.maxlength= 32;
if (blk_bind(blkdesc, FIELD_Prefix, &datafmt, prefix_fieldBuf, prefix_bufLen, prefix_indBuf) != CS_SUCCEED)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
blk_drop(blkdesc);
return -1;
}
//
datafmt.datatype = CS_DATETIME_TYPE;
datafmt.format = CS_FMT_UNUSED;
datafmt.maxlength= CS_SIZEOF(CS_DATETIME);
if (blk_bind(blkdesc, FIELD_UpdateTime, &datafmt, updateTime_fieldBuf, updateTime_bufLen, updateTime_indBuf) != CS_SUCCEED)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
blk_drop(blkdesc);
return -1;
}
//
datafmt.datatype = CS_TINYINT_TYPE;
datafmt.format = CS_FMT_UNUSED;
datafmt.maxlength= CS_SIZEOF(CS_TINYINT);
if (blk_bind(blkdesc, FIELD_SARNumber, &datafmt, sarNumber_fieldBuf, sarNumber_bufLen, sarNumber_indBuf) != CS_SUCCEED)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
blk_drop(blkdesc);
return -1;
}
//
if (blk_bind(blkdesc, FIELD_ExchangeHouse, &datafmt, exchangeHouse_fieldBuf, exchangeHouse_bufLen, exchangeHouse_indBuf) != CS_SUCCEED)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
blk_drop(blkdesc);
return -1;
}
//if (blk_rowxfer_mult(blkdesc, &numrows) == CS_FAIL)
while( (retcode = blk_rowxfer_mult(blkdesc, &numrows)) == CS_SUCCEED ||
retcode == CS_ROW_FAIL)
{
copiedRowCnt += numrows;
//
for(int ii = 0; ii < numrows; ++ii)
{
//
stNumPrefix prefix;
prefix.prefixLen = strlen(prefix_fieldBuf[ii]);
memcpy(prefix.szPrefix, prefix_fieldBuf[ii], prefix.prefixLen);
prefix.areaCodeLen = strlen(areaCode_fieldBuf[ii]);
memcpy(prefix.szAreaCode, areaCode_fieldBuf[ii], prefix.areaCodeLen);
prefix.adminCenterId = sarNumber_fieldBuf[ii];
prefix.switchCenterId = exchangeHouse_fieldBuf[ii];
m_pParentThread->addNumPrefix(prefix);
}
}
//
if(retcode != CS_END_DATA)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
retcode = CS_FAIL;
}
/*
** ALL the rows sent so clear up
*/
CS_INT blktype = copiedRowCnt > 0 ? CS_BLK_ALL : CS_BLK_CANCEL;
if ( (retcode = blk_done(blkdesc, blktype, &numrows)) == CS_FAIL)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
}
if ( (retcode = blk_drop(blkdesc)) == CS_FAIL)
{
DIAG_CLIENT_MESSAGE(connection);
DIAG_SERVER_MESSAGE(connection);
}
return retcode == CS_SUCCEED ? copiedRowCnt : -1;
}
