int deduplicate(struct asfd *asfd, struct conf *conf)
{
struct blk *blk;
struct incoming *in=asfd->in;
struct candidate *champ;
struct candidate *champ_last=NULL;
int count=0;
int blk_count=0;
if(!in) return 0;
incoming_found_reset(in);
count=0;
while((champ=candidates_choose_champ(in, champ_last)))
{
// printf("Got champ: %s %d\n", champ->path, *(champ->score));
if(hash_load(champ->path, conf)) return -1;
if(++count==CHAMPS_MAX) break;
champ_last=champ;
}
blk_count=0;
for(blk=asfd->blist->blk_to_dedup; blk; blk=blk->next)
{
//printf("try: %lu\n", blk->index);
blk_count++;
if(blk_is_zero_length(blk))
{
//printf("got: %s %s\n", blk->weak, blk->strong);
blk->got=BLK_GOT;
in->got++;
continue;
}
// If already got, this function will set blk->save_path
// to be the location of the already got block.
if(already_got_block(asfd, blk)) return -1;
//printf("after agb: %lu %d\n", blk->index, blk->got);
}
logp("%s: %04d/%04d - %04d/%04d\n",
asfd->desc, count, candidates_len, in->got, blk_count);
//cntr_add_same_val(conf->cntr, CMD_DATA, in->got);
// Start the incoming array again.
in->size=0;
// Destroy the deduplication hash table.
hash_delete_all();
asfd->blist->blk_to_dedup=NULL;
return 0;
}
/**
* Call this function to make the object unusable.
* <p>
* {@link ::npppd_auth_base} objects is refered by the {@link ::npppd_ppp}
* object. After this funcation is called, npppd will disconnect the PPP
* links that refers the object, it will call {@link ::npppd_auth_destroy()}
* when all the references to the object are released.</p>
*/
void
npppd_auth_dispose(npppd_auth_base *base)
{
base->disposing = 1;
if (base->users_hash != NULL)
hash_delete_all(base->users_hash, 1);
return;
}
/*
* Empty RelationAccessHash, without destroying the hash table itself.
*/
void
ResetRelationAccessHash()
{
hash_delete_all(RelationAccessHash);
}
static
#endif
int do_backup_phase2_server_protocol2(struct async *as, struct asfd *chfd,
struct sdirs *sdirs, int resume, struct conf **confs)
{
int ret=-1;
uint8_t end_flags=0;
struct slist *slist=NULL;
struct iobuf wbuf;
struct dpth *dpth=NULL;
man_off_t *p1pos=NULL;
struct manios *manios=NULL;
// This is used to tell the client that a number of consecutive blocks
// have been found and can be freed.
struct asfd *asfd=NULL;
struct cntr *cntr=NULL;
struct sbuf *csb=NULL;
uint64_t file_no=1;
if(!as)
{
logp("async not provided to %s()\n", __func__);
goto end;
}
if(!sdirs)
{
logp("sdirs not provided to %s()\n", __func__);
goto end;
}
if(!confs)
{
logp("confs not provided to %s()\n", __func__);
goto end;
}
asfd=as->asfd;
if(!asfd)
{
logp("asfd not provided to %s()\n", __func__);
goto end;
}
if(!chfd)
{
logp("chfd not provided to %s()\n", __func__);
goto end;
}
cntr=get_cntr(confs);
if(get_int(confs[OPT_BREAKPOINT])>=2000
&& get_int(confs[OPT_BREAKPOINT])<3000)
{
breaking=get_int(confs[OPT_BREAKPOINT]);
breakcount=breaking-2000;
}
blks_generate_init();
logp("Phase 2 begin (recv backup data)\n");
if(!(dpth=dpth_alloc())
|| dpth_protocol2_init(dpth,
sdirs->data,
get_string(confs[OPT_CNAME]),
sdirs->cfiles,
get_int(confs[OPT_MAX_STORAGE_SUBDIRS])))
goto end;
if(resume)
{
if(!(p1pos=do_resume(sdirs, dpth, confs)))
goto end;
if(cntr_send_sdirs(asfd, sdirs, confs, CNTR_STATUS_BACKUP))
goto end;
}
if(!(manios=manios_open_phase2(sdirs, p1pos, PROTO_2))
|| !(slist=slist_alloc())
|| !(csb=sbuf_alloc(PROTO_2)))
goto end;
iobuf_free_content(asfd->rbuf);
memset(&wbuf, 0, sizeof(struct iobuf));
while(!(end_flags&END_BACKUP))
{
if(write_status(CNTR_STATUS_BACKUP,
csb && csb->path.buf?csb->path.buf:"", cntr))
goto end;
if(maybe_add_from_scan(manios, slist, chfd, &csb, cntr))
goto end;
if(!wbuf.len)
{
if(get_wbuf_from_sigs(&wbuf, slist, &end_flags))
goto end;
if(!wbuf.len)
{
get_wbuf_from_files(&wbuf, slist,
manios, &end_flags, &file_no);
}
}
if(wbuf.len
&& asfd->append_all_to_write_buffer(asfd, &wbuf)==APPEND_ERROR)
goto end;
if(append_for_champ_chooser(chfd, slist->blist, end_flags))
goto end;
if(as->read_write(as))
{
logp("error from as->read_write in %s\n", __func__);
goto end;
}
while(asfd->rbuf->buf)
{
if(deal_with_read(asfd->rbuf, slist, cntr,
&end_flags, dpth))
goto end;
// Get as much out of the readbuf as possible.
if(asfd->parse_readbuf(asfd))
goto end;
}
while(chfd->rbuf->buf)
{
if(deal_with_read_from_chfd(chfd,
slist->blist, dpth, cntr))
goto end;
// Get as much out of the readbuf as possible.
if(chfd->parse_readbuf(chfd))
goto end;
}
if(write_to_changed_file(asfd, chfd, manios,
slist, end_flags))
goto end;
}
// Hack: If there are some entries left after the last entry that
// contains block data, it will not be written to the changed file
// yet because the last entry of block data has not had
// sb->protocol2->bend set.
if(slist->head && slist->head->next)
{
struct sbuf *sb=NULL;
sb=slist->head;
slist->head=sb->next;
sbuf_free(&sb);
if(write_to_changed_file(asfd, chfd, manios,
slist, end_flags))
goto end;
}
if(manios_close(&manios))
goto end;
if(check_for_missing_work_in_slist(slist))
goto end;
// Need to release the last left. There should be one at most.
if(dpth->head && dpth->head->next)
{
logp("ERROR: More data locks remaining after: %s\n",
dpth->head->save_path);
goto end;
}
if(dpth_release_all(dpth)) goto end;
ret=0;
end:
if(ret)
{
if(slist && slist->head)
logp(" last tried file: %s\n",
iobuf_to_printable(&slist->head->path));
}
logp("End backup\n");
sbuf_free(&csb);
slist_free(&slist);
if(asfd) iobuf_free_content(asfd->rbuf);
if(chfd) iobuf_free_content(chfd->rbuf);
dpth_free(&dpth);
manios_close(&manios);
man_off_t_free(&p1pos);
blks_generate_free();
hash_delete_all();
return ret;
}
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd,
struct asfd *chfd, struct manios *manios,
struct slist *slist, int end_flags)
{
int ret=-1;
struct blk *blk;
static struct iobuf wbuf;
struct blist *blist=slist->blist;
static int unrequested=0;
if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST))
{
if(manio_write_sbuf(manios->changed, sb)) goto end;
sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST;
}
while((blk=sb->protocol2->bstart)
&& blk->got==BLK_GOT
&& (blk->next || end_flags&END_BACKUP))
{
if(blk->got_save_path
&& !blk_is_zero_length(blk))
{
if(breaking && breakcount--==0)
{
breakpoint(breaking, __func__);
goto end;
}
if(manio_write_sig_and_path(manios->changed, blk))
goto end;
if(manios->changed->sig_count==0)
{
// Have finished a manifest file. Want to start
// using it as a dedup candidate now.
if(manio_component_to_chfd(chfd,
manios->changed->offset->ppath))
goto end;
// The champ chooser has the candidate. Now,
// empty our local hash table.
hash_delete_all();
// Add the most recent block, so identical
// adjacent blocks are deduplicated well.
if(hash_load_blk(blk))
goto end;
}
}
if(!blk->requested)
unrequested++;
if(unrequested>BLKS_MAX_UNREQUESTED)
{
unrequested=0;
// Let the client know that it can free memory if there
// was a long consecutive number of unrequested blocks.
get_wbuf_from_index(&wbuf, blk->index);
if(asfd->write(asfd, &wbuf))
goto end;
}
if(blk==sb->protocol2->bend)
{
blist_adjust_head(blist, sb);
if(write_endfile(sb, manios)) return -1;
slist_advance(slist);
return 1;
}
if(sb->protocol2->bsighead==sb->protocol2->bstart)
sb->protocol2->bsighead=blk->next;
sb->protocol2->bstart=blk->next;
if(blk==blist->blk_from_champ_chooser)
blist->blk_from_champ_chooser=blk->next;
}
if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP))
{
// Write endfile for the very last file.
if(write_endfile(sb, manios)) return -1;
}
ret=0;
end:
blist_adjust_head(blist, sb);
return ret;
}
/** Reload the account list */
static int
npppd_auth_reload_acctlist(npppd_auth_base *base)
{
CSVREADER_STATUS status;
int linno, ncols, usersz, nuser, eof, off;
const char **cols, *passwd, *callnum;
char line[8192];
csvreader *csv;
npppd_auth_user *user;
struct in_addr ip4, ip4mask;
slist users;
FILE *file;
struct stat st;
if (base->acctlist_ready != 0 && lstat(base->acctlist_path, &st) == 0) {
if (st.st_mtime == base->last_load)
return 0;
base->last_load = st.st_mtime;
}
slist_init(&users);
csv = NULL;
if ((file = priv_fopen(base->acctlist_path)) == NULL) {
/* hash is empty if file is not found. */
if (errno == ENOENT)
hash_delete_all(base->users_hash, 1);
npppd_auth_base_log(base,
(errno == ENOENT)? LOG_DEBUG : LOG_ERR,
"Open %s failed: %m", base->acctlist_path);
return 0;
}
if ((csv = csvreader_create()) == NULL) {
npppd_auth_base_log(base, LOG_ERR,
"Loading a account list failed: csvreader_create(): %m");
goto fail;
}
for (linno = 0, eof = 0; !eof;) {
ip4.s_addr = 0;
ip4mask.s_addr = 0xffffffffL;
if (fgets(line, sizeof(line), file) != NULL) {
int linelen;
linelen = strlen(line);
if (linelen <= 0) {
npppd_auth_base_log(base, LOG_ERR,
"Loading a account list failed: lineno=%d "
"line too short", linno + 1);
goto fail;
}
if (line[linelen - 1] != '\n' && !feof(file)) {
npppd_auth_base_log(base, LOG_ERR,
"Loading a account list failed: lineno=%d "
"line too long", linno + 1);
goto fail;
}
status = csvreader_parse(csv, line);
} else {
if (!feof(file)) {
npppd_auth_base_log(base, LOG_ERR,
"Loading a account list failed: %m");
goto fail;
}
status = csvreader_parse_flush(csv);
eof = 1;
}
if (status != CSVREADER_NO_ERROR) {
if (status == CSVREADER_OUT_OF_MEMORY)
npppd_auth_base_log(base, LOG_ERR,
"Loading a account list failed: %m");
else
npppd_auth_base_log(base, LOG_ERR,
"Loading a account list "
"failed: lineno=%d parse error", linno);
goto fail;
}
ncols = csvreader_get_number_of_column(csv);
if ((cols = csvreader_get_column(csv)) == NULL)
continue;
linno++; /* count up here because line number is treated as CSV. */
if (linno == 1) {
/* skip a title line */
continue;
}
if (ncols < 1) {
npppd_auth_base_log(base, LOG_ERR,
"account list lineno=%d has only %d fields.",
linno, ncols);
continue;
}
if (strlen(cols[0]) <= 0)
continue; /* skip if the user-name is empty */
if (ncols >= 3) {
if (*cols[2] != '\0' && inet_aton(cols[2], &ip4) != 1) {
npppd_auth_base_log(base, LOG_ERR,
"account list lineno=%d parse error: "
"invalid 'Framed-IP-Address' field: %s",
linno, cols[2]);
continue;
}
}
if (ncols >= 4) {
if ((*cols[3] != '\0' &&
inet_aton(cols[3], &ip4mask) != 1) ||
netmask2prefixlen(htonl(ip4mask.s_addr)) < 0) {
npppd_auth_base_log(base, LOG_ERR,
"account list lineno=%d parse error: "
"invalid 'Framed-IP-Netmask' field: %s",
linno, cols[3]);
continue;
}
}
passwd = "";
if (cols[1] != NULL)
passwd = cols[1];
callnum = "";
if (ncols >= 6 && cols[5] != NULL)
callnum = cols[5];
usersz = sizeof(npppd_auth_user);
usersz += strlen(cols[0]) + 1;
usersz += strlen(passwd) + 1;
usersz += strlen(callnum) + 1;
if ((user = malloc(usersz)) == NULL) {
npppd_auth_base_log(base, LOG_ERR,
"Loading a account list failed: %m");
goto fail;
}
memset(user, 0, usersz);
off = 0;
user->username = user->space + off;
off += strlcpy(user->username, cols[0], usersz - off);
++off;
user->password = user->space + off;
off += strlcpy(user->password, passwd, usersz - off);
++off;
user->calling_number = user->space + off;
strlcpy(user->calling_number, callnum, usersz - off);
user->framed_ip_address = ip4;
user->framed_ip_netmask = ip4mask;
slist_add(&users, user);
}
hash_delete_all(base->users_hash, 1);
nuser = 0;
for (slist_itr_first(&users); slist_itr_has_next(&users);) {
user = slist_itr_next(&users);
if (hash_lookup(base->users_hash, user->username) != NULL) {
npppd_auth_base_log(base, LOG_WARNING,
"Record for user='******' is redefined, the first "
"record will be used.", user->username);
free(user);
goto next_user;
}
if (hash_insert(base->users_hash, user->username, user) != 0) {
npppd_auth_base_log(base, LOG_ERR,
"Loading a account list failed: hash_insert(): %m");
goto fail;
}
nuser++;
next_user:
slist_itr_remove(&users);
}
slist_fini(&users);
csvreader_destroy(csv);
fclose(file);
npppd_auth_base_log(base, LOG_INFO,
"Loaded users from='%s' successfully. %d users",
base->acctlist_path, nuser);
base->acctlist_ready = 1;
return 0;
fail:
fclose(file);
if (csv != NULL)
csvreader_destroy(csv);
hash_delete_all(base->users_hash, 1);
for (slist_itr_first(&users); slist_itr_has_next(&users);) {
user = slist_itr_next(&users);
free(user);
}
slist_fini(&users);
return 1;
}
static void tear_down(void)
{
hash_delete_all();
alloc_check();
}
void analyse_data() {
hash_node_type* n = NULL;
if(options.paused == 1) {
return;
}
// Zero totals
memset(&totals, 0, sizeof totals);
if(options.freezeorder) {
screen_hash_clear();
}
else {
screen_list_clear();
hash_delete_all(screen_hash);
}
while(hash_next_item(history, &n) == HASH_STATUS_OK) {
history_type* d = (history_type*)n->rec;
host_pair_line* screen_line;
union {
host_pair_line **h_p_l_pp;
void **void_pp;
} u_screen_line = { &screen_line };
addr_pair ap;
int i;
int tsent, trecv;
tsent = trecv = 0;
ap = *(addr_pair*)n->key;
/* Aggregate hosts, if required */
if(options.aggregate_src) {
memset(&ap.src6, '\0', sizeof(ap.src6));
}
if(options.aggregate_dest) {
memset(&ap.dst6, '\0', sizeof(ap.dst6));
}
/* Aggregate ports, if required */
if(options.showports == OPTION_PORTS_DEST || options.showports == OPTION_PORTS_OFF) {
ap.src_port = 0;
}
if(options.showports == OPTION_PORTS_SRC || options.showports == OPTION_PORTS_OFF) {
ap.dst_port = 0;
}
if(options.showports == OPTION_PORTS_OFF) {
ap.protocol = 0;
}
if(hash_find(screen_hash, &ap, u_screen_line.void_pp) == HASH_STATUS_KEY_NOT_FOUND) {
screen_line = xcalloc(1, sizeof *screen_line);
hash_insert(screen_hash, &ap, screen_line);
screen_line->ap = ap;
}
screen_line->total_sent += d->total_sent;
screen_line->total_recv += d->total_recv;
for(i = 0; i < HISTORY_LENGTH; i++) {
int j;
int ii = (HISTORY_LENGTH + history_pos - i) % HISTORY_LENGTH;
for(j = 0; j < HISTORY_DIVISIONS; j++) {
if(i < history_divs[j]) {
screen_line->recv[j] += d->recv[ii];
screen_line->sent[j] += d->sent[ii];
}
}
}
}
make_screen_list();
calculate_totals();
}
