/*
fp->sp->owner is scp (assert this is non-NULL)
fp->sp is sp
sigs[0] is sig
*/
void
sigs_cache_add(struct ilem_form *ifp, struct sig const *const *sigs)
{
if (!ifp || !ifp->sp || BIT_ISSET(ifp->f2.flags, F2_FLAGS_FROM_CACHE))
return;
if (!ifp->sp->cache)
sigs_cache_init(ifp->sp);
if (verbose)
fprintf(stderr, "sigs_cache: adding %s to cache\n", ifp->f2.form);
sigs_load_one_sig(ifp->sp->owner, ifp->sp->cache, sigs[0]->sig, 0, ifp);
#if 0
if (BIT_ISSET(fp->f2.flags, F2_FLAGS_LEM_BY_NORM))
hash_add(fp->sp->cache,
npool_copy(ifp->f2.norm, sigs_cache_pool),
(void*)sigs);
else
hash_add(fp->sp->cache,
npool_copy(ifp->f2.form, sigs_cache_pool),
(void*)sigs);
#endif
}
static void
sprite_decode(sprite_t *sprite, const memory_oam_t* oam)
{
sprite->y = MAX(0, oam->data[0] - SPRITE_HEIGHT);
sprite->x = MAX(0, oam->data[1] - SPRITE_WIDTH);
sprite->tile_y = MAX(0, SPRITE_HEIGHT - oam->data[0]);
sprite->tile_x = MAX(0, SPRITE_WIDTH - oam->data[1]);
if (sprite->tile_y < TILE_HEIGHT) {
sprite->tile_id = oam->data[2] & 0xFE;
} else {
sprite->tile_id = oam->data[2] | 0x01;
}
sprite->visible = false;
if (sprite->tile_x < SPRITE_WIDTH && sprite->tile_y < SPRITE_HEIGHT) {
sprite->visible = true;
}
sprite->tile_y %= TILE_HEIGHT;
sprite->in_background = BIT_ISSET(oam->data[2], 7);
sprite->flip_y = BIT_ISSET(oam->data[2], 6);
sprite->flip_x = BIT_ISSET(oam->data[2], 5);
sprite->high_palette = BIT_ISSET(oam->data[2], 4);
}
/* read a header block */
int
th_read_internal(TAR *t)
{
int i;
int num_zero_blocks = 0;
#ifdef DEBUG
printf("==> th_read_internal(TAR=\"%s\")\n", t->pathname);
#endif
while ((i = tar_block_read(t, &(t->th_buf))) == T_BLOCKSIZE)
{
/* two all-zero blocks mark EOF */
if (t->th_buf.name[0] == '\0')
{
num_zero_blocks++;
if (!BIT_ISSET(t->options, TAR_IGNORE_EOT)
&& num_zero_blocks >= 2)
return 0; /* EOF */
else
continue;
}
/* verify magic and version */
if (BIT_ISSET(t->options, TAR_CHECK_MAGIC)
&& strncmp(t->th_buf.magic, TMAGIC, TMAGLEN - 1) != 0)
{
#ifdef DEBUG
puts("!!! unknown magic value in tar header");
#endif
return -2;
}
if (BIT_ISSET(t->options, TAR_CHECK_VERSION)
&& strncmp(t->th_buf.version, TVERSION, TVERSLEN) != 0)
{
#ifdef DEBUG
puts("!!! unknown version value in tar header");
#endif
return -2;
}
/* check chksum */
if (!BIT_ISSET(t->options, TAR_IGNORE_CRC)
&& !th_crc_ok(t))
{
#ifdef DEBUG
puts("!!! tar header checksum error");
#endif
return -2;
}
break;
}
#ifdef DEBUG
printf("<== th_read_internal(): returning %d\n", i);
#endif
return i;
}
int
check_and_register(const char *id, int set_ok)
{
unsigned char *vec;
unsigned int vmax;
long idnum = strtoul(id+1,NULL,10);
unsigned int v, b;
int tab[8] = { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 };
/* The vector is created by Perl's vec() function; perldoc -f vec for more info */
v = idnum/8;
b = tab[idnum%8];
if (*id == 'P')
{
vec = pvec;
vmax = 1000000;
}
else if (*id == 'Q')
{
vec = qvec;
vmax = QVEC_SIZ;
}
else
return 0;
if (idnum < vmax && set_ok)
{
if (BIT_ISSET(vec[v],b))
{
vwarning("duplicate ID: %s", id);
return 1;
}
else
{
BIT_SET(vec[v],b);
}
}
#if 1
/* issue this as a notice so it doesn't affect error status */
if (*id == 'P' && (idnum > max_cat_id || !(BIT_ISSET(catvec[v],b))))
{
if (strlen(id) > 7)
{
vwarning("%s: malformed ID; text will be ignored",id);
return 1;
}
else
vnotice("%s not in main catalog (new P-IDs may not register for 48 hours)",id);
}
#else
else
{
vwarning("%s: ID too big (max %d)",id,vmax);
return 1;
}
#endif
return 0;
}
static void
inherit_f2(struct f2 *inheritor_f2, struct f2 *from_f2)
{
#define inherit(memb) if((!(inheritor_f2->memb) || !(*inheritor_f2->memb))&&from_f2->memb)inheritor_f2->memb=from_f2->memb
if (BIT_ISSET(inheritor_f2->flags, F2_FLAGS_FROM_CACHE))
{
*inheritor_f2 = *from_f2;
return;
}
#if 0
if (!strcmp((const char *)inheritor_f2->form, "*"))
inherit(cf);
#endif
inherit(base);
inherit(cont);
inherit(norm);
/* Fix the CF and GW fields: in L2 we can't make any kind
of a match without these either matching CF/NORM or
GW/SENSE, so this coercion is safe */
if (!inheritor_f2->cf || (!BIT_ISSET(inheritor_f2->flags, F2_FLAGS_NOT_IN_SIGS)
&& strcmp((char*)inheritor_f2->cf,(char*)from_f2->cf)))
{
inheritor_f2->cf = from_f2->cf;
BIT_CLEAR(inheritor_f2->flags, F2_FLAGS_NORM_IS_CF);
}
if (!inheritor_f2->gw || (!BIT_ISSET(inheritor_f2->flags, F2_FLAGS_NOT_IN_SIGS)
&& strcmp((char*)inheritor_f2->gw,(char*)from_f2->gw)))
inheritor_f2->gw = from_f2->gw;
if (!inheritor_f2->sense || (!BIT_ISSET(inheritor_f2->flags, F2_FLAGS_NOT_IN_SIGS)
&& strcmp((char*)inheritor_f2->sense,(char*)from_f2->sense)))
inheritor_f2->sense = from_f2->sense;
if (!inheritor_f2->pos)
inheritor_f2->pos = from_f2->pos;
else
{
if (!BIT_ISSET(inheritor_f2->flags, F2_FLAGS_NOT_IN_SIGS))
{
if (strcmp((char*)inheritor_f2->pos,(char*)from_f2->pos))
{
/* silently correct x[y]N where it is really x[y]'N */
inheritor_f2->epos = inheritor_f2->pos;
inheritor_f2->pos = from_f2->pos;
}
}
}
inherit(epos);
inherit(stem);
inherit(morph);
inherit(morph2);
#undef inherit
}
void
sigs_dump_sigs(struct xcl_context *xcp, struct xcl_l *lp)
{
if (lp && lp->f && lp->f->f2.pos && strcmp(lp->f->f2.pos,"X")
&& !BIT_ISSET(lp->f->f2.flags,F2_FLAGS_FROM_CACHE)
&& ((lp->f->f2.base && *lp->f->f2.base)
|| !BIT_ISSET(lp->f->f2.core->features,LF_BASE))
&& lp->f->f2.sig)
fprintf(stdout,"%s\n", lp->f->f2.sig);
}
/* install package */
int
install_pkg(char *pkg)
{
ENCAP *encap;
int status;
if (verbose)
printf(" > installing package %s\n", pkg);
if (encap_open(&encap, source, target, pkg,
options, epkg_print) == -1)
{
printf(" ! cannot open package %s - installation aborted\n",
pkg);
return -1;
}
status = encap_install(encap, override_decision);
encap_close(encap);
/* write logfile */
if (status != ENCAP_STATUS_NOOP &&
!BIT_ISSET(options, OPT_SHOWONLY) &&
BIT_ISSET(epkg_opts, EPKG_OPT_WRITELOG))
write_encap_log(pkg, MODE_INSTALL, status);
switch (status)
{
case ENCAP_STATUS_FAILED:
if (verbose)
printf(" ! installation failed\n");
break;
case ENCAP_STATUS_SUCCESS:
if (verbose)
printf(" > installation successful\n");
break;
case ENCAP_STATUS_PARTIAL:
if (verbose)
printf(" > installation partially successful\n");
break;
case ENCAP_STATUS_NOOP:
break;
default:
printf(" ! unknown status %d\n", status);
return -1;
}
if (status == ENCAP_STATUS_FAILED ||
(status != ENCAP_STATUS_NOOP && BIT_ISSET(options, OPT_SHOWONLY)))
return 1;
return 0;
}
static int handleDisconnect1(struct slice *slice, int clNo)
{
if(slice != NULL) {
if (BIT_ISSET(clNo, slice->sl_reqack.readySet)) {
/* avoid counting client both as left and ready */
CLR_BIT(clNo, slice->sl_reqack.readySet);
slice->nrReady--;
}
if (BIT_ISSET(clNo, slice->answeredSet)) {
slice->nrAnswered--;
CLR_BIT(clNo, slice->answeredSet);
}
}
return 0;
}
void _os_task_delay_starter(void *arg) {
int x;
while(1) {
os_sleep(1000);
uptime_secs++;
for(x = 0; x < num_tasks; x++) {
if(!BIT_ISSET(tasks[x].flags, TASK_FLAG_RUNNING) && !BIT_ISSET(tasks[x].flags, TASK_FLAG_DONE) &&
tasks[x].start_delay_secs > 0) {
tasks[x].start_delay_secs -= 1;
}
}
}
}
int start_task(void *addr, void *arg, uint16_t start_delay) {
if(num_tasks == MAX_TASKS) {
// search for a task that is marked as done, take it over
int x;
for(x = 0; x < num_tasks; ++x) {
if(BIT_ISSET(tasks[x].flags, TASK_FLAG_DONE)) {
tasks[x].address = addr;
tasks[x].arg = arg;
tasks[x].start_delay_secs = start_delay;
// reset flags
// running = 0
// done = 0
tasks[x].flags = 0;
tasks[x].saved_sp = NULL;
tasks[x].delayMillis = 0;
return 0;
}
}
return -1;
}
else {
tasks[num_tasks].address = addr;
tasks[num_tasks].arg = arg;
tasks[num_tasks].start_delay_secs = start_delay;
num_tasks++;
return 0;
}
}
unsigned char getchar(void)
{
int doloop = 0;
int k;
unsigned char c;
do {
do {
c=getc();
if (c == KEY_SHIFT) {
BIT_SET(key_statbits, KEY_STATBIT_SHIFT);
}
} while (! (c & 0x80) || c==KEY_SHIFT);
k = c & 0x7f;
if (k == KEY_SHIFT) {
BIT_USET(key_statbits, KEY_STATBIT_SHIFT);
doloop = 1;
} else {
if (KEY_1 <= k && KEY_0 >= k) {
k = (((k - 1)%10) + 0x30);
} else if (BIT_ISSET(key_statbits, KEY_STATBIT_SHIFT) ) {
/* char upper case */
}
doloop = 0;
}
} while (doloop==1);
return k;
}
/*
* Verifies freed addresses. Checks for alignment, valid slab membership
* and duplicate frees.
*
*/
void
uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item)
{
uma_keg_t keg;
int freei;
if (zone_first_keg(zone) == NULL)
return;
if (slab == NULL) {
slab = uma_dbg_getslab(zone, item);
if (slab == NULL)
panic("uma: Freed item %p did not belong to zone %s\n",
item, zone->uz_name);
}
keg = slab->us_keg;
freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
if (freei >= keg->uk_ipers)
panic("Invalid free of %p from zone %p(%s) slab %p(%d)\n",
item, zone, zone->uz_name, slab, freei);
if (((freei * keg->uk_rsize) + slab->us_data) != item)
panic("Unaligned free of %p from zone %p(%s) slab %p(%d)\n",
item, zone, zone->uz_name, slab, freei);
if (!BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree))
panic("Duplicate free of %p from zone %p(%s) slab %p(%d)\n",
item, zone, zone->uz_name, slab, freei);
BIT_CLR_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree);
}
/**
* Handles ok message
*/
static int handleOk(sender_state_t sendst,
struct slice *slice,
int clNo)
{
if(slice == NULL)
return 0;
if(!udpc_isParticipantValid(sendst->rc.participantsDb, clNo)) {
udpc_flprintf("Invalid participant %d\n", clNo);
return 0;
}
if (BIT_ISSET(clNo, slice->sl_reqack.readySet)) {
/* client is already marked ready */
#if DEBUG
flprintf("client %d is already ready\n", clNo);
#endif
} else {
SET_BIT(clNo, slice->sl_reqack.readySet);
slice->nrReady++;
#if DEBUG
flprintf("client %d replied ok for %p %d ready=%d\n", clNo,
slice, slice->sliceNo, slice->nrReady);
#endif
senderSetAnswered(sendst->stats, clNo);
markParticipantAnswered(slice, clNo);
}
return 0;
}
static void
lem_f2_serialize(FILE *fp, struct f2 *f2)
{
if (BIT_ISSET(f2->flags, F2_FLAGS_NOT_IN_SIGS))
{
if (f2->pos)
{
fputs((char*)f2->pos,fp);
}
else
fputs("X",fp);
}
else
{
if (f2->norm || f2->cf)
{
if (strncmp((char*)f2->lang,"sux",3) && f2->norm)
fputs((char*)f2->norm,fp);
else
fputs((char*)f2->cf,fp);
if (f2->sense)
fprintf(fp,"[%s]",(char*)f2->sense);
else
fprintf(fp,"[%s]",f2->gw);
if (f2->epos && strcmp((char*)f2->pos,(char*)f2->epos))
fprintf(fp,"'%s",f2->epos);
else
fputs((char*)f2->pos,fp);
}
else
{
fputs("X",fp);
}
}
}
static void markParticipantAnswered(slice_t slice, int clNo)
{
if(BIT_ISSET(clNo, slice->answeredSet))
/* client already has answered */
return;
slice->nrAnswered++;
SET_BIT(clNo, slice->answeredSet);
}
bool FastqParserExt::ReadNextRecord(FastqRecord &rec_, uchar *tagBuffer_, uint64 tagPreserveFlags_)
{
const char *fieldSeparators = " ._,=:/-#"; //9
//const std::vector<uchar> separators(fieldSeparators, fieldSeparators + 9 + 1);
if (memoryPos == memorySize)
return false;
rec_.title = memory + memoryPos;
rec_.titleLen = SkipLine();
if (rec_.titleLen == 0 || rec_.title[0] != '@')
return false;
ASSERT(rec_.titleLen <= MaxTagBufferSize);
uint32 fieldNo = 0;
uint32 fieldBeginPos = 0;
uint32 bufferPos = 0;
for (uint32 i = 0; i <= rec_.titleLen; ++i)
{
if (!std::count(fieldSeparators, fieldSeparators + 10, rec_.title[i]) && (i != rec_.titleLen))
continue;
fieldNo++;
if (BIT_ISSET(tagPreserveFlags_, fieldNo))
{
std::copy(rec_.title + fieldBeginPos, rec_.title + i + 1, tagBuffer_ + bufferPos);
bufferPos += (i + 1 - fieldBeginPos);
}
fieldBeginPos = i + 1;
}
ASSERT(rec_.titleLen >= bufferPos);
totalBytesCut += rec_.titleLen - bufferPos;
if (bufferPos > 0)
{
std::copy(tagBuffer_, tagBuffer_ + bufferPos, rec_.title);
}
rec_.titleLen = bufferPos;
rec_.sequence = memory + memoryPos;
rec_.sequenceLen = SkipLine();
// read plus
uint32 plusLen = SkipLine();
rec_.quality = memory + memoryPos;
rec_.qualityLen = SkipLine();
return (plusLen > 0 && rec_.sequenceLen == rec_.qualityLen);
}
unsigned char *
f2_psu_sig(struct f2 *fp, struct npool *pool)
{
unsigned char buf[1024];
#if 1
sprintf((char*)buf,"{%s}::",fp->psu_ngram);
#else
sprintf((char*)buf,"{%s[%s//%s]%s'%s",
fp->cf ? fp->cf : (Uchar*)"X",
fp->gw ? fp->gw : (Uchar*)"X",
fp->sense ? fp->sense : (Uchar*)"X",
fp->pos ? fp->pos : (Uchar*)"X",
fp->epos ? fp->epos : (Uchar*)"X");
strcat((char*)buf, "}::");
#endif
if (fp->parts)
{
int i;
char *amp = NULL;
for (i = 0; fp->parts[i]; ++i)
{
if (i)
strcat((char*)buf, "++");
#if 1
if (fp->parts[i]->tail_sig)
strcat((char*)buf, (char*)fp->parts[i]->tail_sig);
else
{
if (!fp->parts[i]->sig)
fp->parts[i]->sig = f2_sig(fp->parts[i], pool);
if ((amp = strstr((char*)fp->parts[i]->sig, "&&")))
{
int len = strlen((char*)buf) + (amp - (char*)fp->parts[i]->sig);
strncat((char*)buf, (char*)fp->parts[i]->sig, amp - (char*)fp->parts[i]->sig);
buf[len] = '\0';
}
else
strcat((char*)buf,tabless(fp->parts[i]->sig));
#else
if (BIT_ISSET(fp->parts[i]->flags, F2_FLAGS_SAME_REF))
append_sig_sans_form(buf,(unsigned char*)tabless(fp->parts[i]->sig));
else
strcat((char*)buf,tabless(fp->parts[i]->sig));
#endif
}
}
}
return npool_copy(buf,pool);
}
int moloch_http_curl_close_callback(void *serverV, curl_socket_t fd)
{
MolochHttpServer_t *server = serverV;
if (! BIT_ISSET(fd, connectionsSet)) {
LOG("Couldn't connect %s defaultPort: %d", server->names[0], server->defaultPort);
return 0;
}
struct sockaddr_in localAddress, remoteAddress;
memset(&localAddress, 0, sizeof(localAddress));
memset(&remoteAddress, 0, sizeof(localAddress));
socklen_t addressLength = sizeof(localAddress);
getsockname(fd, (struct sockaddr*)&localAddress, &addressLength);
addressLength = sizeof(remoteAddress);
getpeername(fd, (struct sockaddr*)&remoteAddress, &addressLength);
char sessionId[MOLOCH_SESSIONID_LEN];
moloch_session_id(sessionId, localAddress.sin_addr.s_addr, localAddress.sin_port,
remoteAddress.sin_addr.s_addr, remoteAddress.sin_port);
MolochHttpConn_t *conn;
BIT_CLR(fd, connectionsSet);
MOLOCH_LOCK(connections);
HASH_FIND(h_, connections, sessionId, conn);
if (conn) {
HASH_REMOVE(h_, connections, conn);
MOLOCH_TYPE_FREE(MolochHttpConn_t, conn);
}
MOLOCH_UNLOCK(connections);
server->connections--;
LOG("Close %d/%d - %s %d->%s:%d fd:%d",
server->outstanding,
server->connections,
server->names[0],
ntohs(localAddress.sin_port),
inet_ntoa(remoteAddress.sin_addr),
ntohs(remoteAddress.sin_port),
fd);
close (fd);
return 0;
}
int
override_decision(ENCAP *encap, encap_source_info_t *srcinfo,
encap_target_info_t *tgtinfo)
{
encap_listptr_t lp;
int i;
#ifdef DEBUG
printf("==> override_decision(encap=0x%lx, srcinfo=\"%s\", "
"tgtinfo=\"%s\")\n", encap, srcinfo->src_pkgdir_relative,
tgtinfo->tgt_link_existing_pkgdir_relative);
#endif
i = exclude_decision(encap, srcinfo, tgtinfo);
if (i != R_FILEOK)
return i;
encap_listptr_reset(&lp);
if (BIT_ISSET(tgtinfo->tgt_flags, TGT_DEST_ENCAP_SRC) &&
strcmp(encap->e_pkgname, tgtinfo->tgt_link_existing_pkg) != 0 &&
encap_list_search(override_l, &lp,
tgtinfo->tgt_link_existing_pkg, NULL) != 0)
{
(*encap->e_print_func)(encap, srcinfo, tgtinfo, EPT_PKG_INFO,
"overriding link to package %s",
tgtinfo->tgt_link_existing_pkg);
if (remove(srcinfo->src_target_path) == -1)
{
(*encap->e_print_func)(encap, srcinfo, tgtinfo,
EPT_INST_ERROR, "remove");
return R_ERR;
}
if (encap_check_target(encap->e_source,
srcinfo->src_target_path, tgtinfo) == -1)
{
(*encap->e_print_func)(encap, srcinfo, tgtinfo,
EPT_INST_ERROR,
"encap_check_target");
return R_ERR;
}
}
return R_FILEOK;
}
void printSet(participantsDb_t db, char *d)
{
int first=1;
int i;
fprintf(stderr, "[");
for (i=0; i < MAX_CLIENTS; i++) {
if (db->clientTable[i].used) {
if(BIT_ISSET(i, d)) {
if(!first)
fprintf(stderr,",");
first=0;
fprintf(stderr, "%d", i);
}
}
}
fprintf(stderr, "]");
}
/* This routine should not set anything but FORM at the f2 level;
that is the job of ilem_parse */
void
lem_save_form(const char *ref, const char *lang,
const char *formstr, struct lang_context *langcon)
{
struct ilem_form *form = mb_new(lemline_xcp->sigs->mb_ilem_forms);
extern int curr_cell;
form->ref = (char*)ref;
if (lang)
{
form->f2.lang = (unsigned char*)lang;
form->f2.core = langcore_of(lang);
if (strstr(lang,"949"))
BIT_SET(form->f2.flags,F2_FLAGS_LEM_BY_NORM);
}
if (BIT_ISSET(form->f2.flags,F2_FLAGS_LEM_BY_NORM))
{
form->f2.norm = (unsigned char *)formstr;
form->f2.form = (const unsigned char *)"*";
}
else
form->f2.form = (unsigned char *)formstr;
form->file = (char*)file;
form->lnum = lnum;
form->lang = langcon;
if (!ref[0])
return;
if (!curr_lsp->forms_alloced
|| curr_lsp->forms_used == curr_lsp->forms_alloced)
{
curr_lsp->forms_alloced += 16;
curr_lsp->forms = realloc(curr_lsp->forms,
curr_lsp->forms_alloced*sizeof(struct ilem_form*));
curr_lsp->cells = realloc(curr_lsp->cells,
curr_lsp->forms_alloced*sizeof(int));
if (curr_lsp->forms_used < 0)
curr_lsp->forms_used = 0;
}
/* when curr_cell = 0 we are in a line with no cells; by definition,
all content in such a line is in cell 2 (because cell 1 is the line
number) */
curr_lsp->cells[curr_lsp->forms_used] = (curr_cell ? curr_cell : 2);
curr_lsp->forms[curr_lsp->forms_used++] = form;
hash_add(word_form_index,npool_copy((unsigned char*)ref,lemline_xcp->pool),form);
}
static int
privcmd_pg_fault(vm_object_t object, vm_ooffset_t offset,
int prot, vm_page_t *mres)
{
struct privcmd_map *map = object->handle;
vm_pindex_t pidx;
vm_page_t page, oldm;
if (map->mapped != true)
return (VM_PAGER_FAIL);
pidx = OFF_TO_IDX(offset);
if (pidx >= map->size || BIT_ISSET(map->size, pidx, map->err))
return (VM_PAGER_FAIL);
page = PHYS_TO_VM_PAGE(map->phys_base_addr + offset);
if (page == NULL)
return (VM_PAGER_FAIL);
KASSERT((page->flags & PG_FICTITIOUS) != 0,
("not fictitious %p", page));
KASSERT(page->wire_count == 1, ("wire_count not 1 %p", page));
KASSERT(vm_page_busied(page) == 0, ("page %p is busy", page));
if (*mres != NULL) {
oldm = *mres;
vm_page_lock(oldm);
vm_page_free(oldm);
vm_page_unlock(oldm);
*mres = NULL;
}
vm_page_insert(page, object, pidx);
page->valid = VM_PAGE_BITS_ALL;
vm_page_xbusy(page);
*mres = page;
return (VM_PAGER_OK);
}
/*
* Set up the slab's freei data such that uma_dbg_free can function.
*
*/
void
uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item)
{
uma_keg_t keg;
int freei;
if (zone_first_keg(zone) == NULL)
return;
if (slab == NULL) {
slab = uma_dbg_getslab(zone, item);
if (slab == NULL)
panic("uma: item %p did not belong to zone %s\n",
item, zone->uz_name);
}
keg = slab->us_keg;
freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
if (BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree))
panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)\n",
item, zone, zone->uz_name, slab, freei);
BIT_SET_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree);
return;
}
bool digitContainsSeg( char c, char seg ) {
return BIT_ISSET( asciiToSegMap( c ), seg_num_to_bit[ seg ] );
}
static int sendSlice(sender_state_t sendst, struct slice *slice,
int retransmitting)
{
struct net_config *config = sendst->config;
int nrBlocks, i;
#ifdef BB_FEATURE_UDPCAST_FEC
int fecBlocks;
#endif
int retransmissions=0;
if(retransmitting) {
slice->nextBlock = 0;
if(slice->state != SLICE_XMITTED)
return 0;
} else {
if(slice->state != SLICE_NEW)
return 0;
}
nrBlocks = getSliceBlocks(slice, config);
#ifdef BB_FEATURE_UDPCAST_FEC
if((config->flags & FLAG_FEC) && !retransmitting) {
fecBlocks = config->fec_redundancy * config->fec_stripes;
} else {
fecBlocks = 0;
}
#endif
#if DEBUG
if(retransmitting) {
flprintf("%s slice %d from %d to %d (%d bytes) %d\n",
retransmitting ? "Retransmitting" : "Sending",
slice->sliceNo, slice->nextBlock, nrBlocks, slice->bytes,
config->blockSize);
}
#endif
/* transmit the data */
for(i = slice->nextBlock; i < nrBlocks
#ifdef BB_FEATURE_UDPCAST_FEC
+ fecBlocks
#endif
; i++) {
if(retransmitting) {
if(!BIT_ISSET(i, slice->rxmitMap) ||
BIT_ISSET(i, slice->isXmittedMap)) {
/* if slice is not in retransmit list, or has _already_
* been retransmitted, skip it */
if(i > slice->lastGoodBlock)
slice->lastGoodBlock = i;
continue;
}
SET_BIT(i, slice->isXmittedMap);
retransmissions++;
#if DEBUG
flprintf("Retransmitting %d.%d\n", slice->sliceNo, i);
#endif
}
if(i < nrBlocks)
transmitDataBlock(sendst, slice, i);
#ifdef BB_FEATURE_UDPCAST_FEC
else
transmitFecBlock(sendst, slice, i - nrBlocks);
#endif
if(!retransmitting && pc_getWaiting(sendst->rc.incoming)) {
i++;
break;
}
}
if(retransmissions)
senderStatsAddRetransmissions(sendst->stats, retransmissions);
slice->nextBlock = i;
if(i == nrBlocks
#ifdef BB_FEATURE_UDPCAST_FEC
+ fecBlocks
#endif
) {
slice->needRxmit = 0;
if(!retransmitting)
slice->state = SLICE_XMITTED;
#if DEBUG
flprintf("Done: at block %d %d %d\n", i, retransmitting,
slice->state);
#endif
return 2;
}
#if DEBUG
flprintf("Done: at block %d %d %d\n", i, retransmitting,
slice->state);
#endif
return 1;
}
int moloch_http_curl_close_callback(void *snameV, curl_socket_t fd)
{
MolochHttpServerName_t *sname = snameV;
MolochHttpServer_t *server = sname->server;
if (! BIT_ISSET(fd, connectionsSet)) {
long ev = (long)g_hash_table_lookup(server->fd2ev, (void *)(long)fd);
LOG("Couldn't connect %s (%d, %ld) ", sname->name, fd, ev);
close(fd);
GSource *source = g_main_context_find_source_by_id (NULL, ev);
if (source)
g_source_destroy (source);
g_hash_table_remove(server->fd2ev, (void *)(long)fd);
return 0;
}
struct sockaddr_storage localAddressStorage, remoteAddressStorage;
socklen_t addressLength = sizeof(localAddressStorage);
int rc = getsockname(fd, (struct sockaddr*)&localAddressStorage, &addressLength);
if (rc != 0)
return 0;
addressLength = sizeof(remoteAddressStorage);
rc = getpeername(fd, (struct sockaddr*)&remoteAddressStorage, &addressLength);
if (rc != 0)
return 0;
char sessionId[MOLOCH_SESSIONID_LEN];
int localPort, remotePort;
char remoteIp[INET6_ADDRSTRLEN+2];
if (localAddressStorage.ss_family == AF_INET) {
struct sockaddr_in *localAddress = (struct sockaddr_in *)&localAddressStorage;
struct sockaddr_in *remoteAddress = (struct sockaddr_in *)&remoteAddressStorage;
moloch_session_id(sessionId, localAddress->sin_addr.s_addr, localAddress->sin_port,
remoteAddress->sin_addr.s_addr, remoteAddress->sin_port);
localPort = ntohs(localAddress->sin_port);
remotePort = ntohs(remoteAddress->sin_port);
inet_ntop(AF_INET, &remoteAddress->sin_addr, remoteIp, sizeof(remoteIp));
} else {
struct sockaddr_in6 *localAddress = (struct sockaddr_in6 *)&localAddressStorage;
struct sockaddr_in6 *remoteAddress = (struct sockaddr_in6 *)&remoteAddressStorage;
moloch_session_id6(sessionId, localAddress->sin6_addr.s6_addr, localAddress->sin6_port,
remoteAddress->sin6_addr.s6_addr, remoteAddress->sin6_port);
localPort = ntohs(localAddress->sin6_port);
remotePort = ntohs(remoteAddress->sin6_port);
inet_ntop(AF_INET6, &remoteAddress->sin6_addr, remoteIp+1, sizeof(remoteIp)-2);
remoteIp[0] = '[';
strcat(remoteIp, "]");
}
MolochHttpConn_t *conn;
BIT_CLR(fd, connectionsSet);
MOLOCH_LOCK(connections);
HASH_FIND(h_, connections, sessionId, conn);
if (conn) {
HASH_REMOVE(h_, connections, conn);
MOLOCH_TYPE_FREE(MolochHttpConn_t, conn);
}
MOLOCH_UNLOCK(connections);
server->connections--;
if (config.logHTTPConnections) {
LOG("Close %d/%d - %s %d->%s:%d fd:%d removed: %s",
server->outstanding,
server->connections,
sname->name,
localPort,
remoteIp,
remotePort,
fd,
conn?"true":"false");
}
close (fd);
return 0;
}
/*
** find_update_versions() - return a list of update_file structs which
** point to the versions of pkgname which are
** found in the update directories
*/
encap_list_t *
find_update_versions(char *pkgname)
{
encap_listptr_t lp;
char *cp;
char update_dir[MAXPATHLEN];
update_dir_t *udp;
update_state_t us;
#ifdef DEBUG
printf("==> find_update_versions(pkgname=\"%s\")\n", pkgname);
#endif
us.us_pkg = pkgname;
us.us_uv_l = encap_list_new(LIST_USERFUNC, (encap_cmpfunc_t)uf_cmp);
if (us.us_uv_l == NULL)
{
fprintf(stderr, " ! encap_list_new() failed\n");
return NULL;
}
encap_listptr_reset(&lp);
while (encap_list_next(update_path_l, &lp) != 0)
{
udp = (update_dir_t *)encap_listptr_data(&lp);
/* replace "%p" token with platform name */
encap_gsub(udp->ud_url, "%p", platform,
update_dir, sizeof(update_dir));
#if 0
/* strip trailing '/' for cosmetic purposes */
sz = strlen(update_dir) - 1;
if (update_dir[sz] == '/')
update_dir[sz] = '\0';
#endif
if (udp->ud_pkgs_l == NULL)
{
udp->ud_pkgs_l = encap_list_new(LIST_QUEUE, NULL);
if (udp->ud_pkgs_l == NULL)
return NULL;
if (download_dir(update_dir,
(dir_entry_func_t)add_update,
udp->ud_pkgs_l) == -1)
{
encap_list_del(update_path_l, &lp);
encap_list_prev(update_path_l, &lp);
}
}
if (encap_list_iterate(udp->ud_pkgs_l,
(encap_iterate_func_t)update_check,
&us) == -1)
return NULL;
/*
** if EPKG_OPT_UPDATEALLDIRS isn't set and we found a
** matching package in this directory, stop here
*/
if (!BIT_ISSET(epkg_opts, EPKG_OPT_UPDATEALLDIRS)
&& encap_list_nents(us.us_uv_l) > 0)
break;
}
if (encap_list_nents(us.us_uv_l) == 0)
{
encap_list_free(us.us_uv_l, NULL);
us.us_uv_l = NULL;
}
return us.us_uv_l;
}
/* check mode */
int
check_mode(char *pkgspec)
{
encap_list_t *ver_l;
char *checkver;
char pkg[MAXPATHLEN], name[MAXPATHLEN], ver[MAXPATHLEN] = "";
encap_listptr_t lp;
int i;
if (verbose)
printf("epkg: checking package %s...\n", pkgspec);
/* if versioning is off, simply install the specified package */
if (!BIT_ISSET(epkg_opts, EPKG_OPT_VERSIONING))
return check_pkg(pkgspec);
if (verbose)
printf(" > reading Encap source directory...\n");
ver_l = encap_list_new(LIST_USERFUNC, (encap_cmpfunc_t)encap_vercmp);
if (ver_l == NULL)
{
fprintf(stderr, " ! encap_list_new(): %s\n",
strerror(errno));
return -1;
}
i = encap_find_versions(source, pkgspec, version_list_add, ver_l);
if (i == 0)
{
encap_pkgspec_parse(pkgspec, name, sizeof(name),
ver, sizeof(ver), NULL, 0, NULL, 0);
i = encap_find_versions(source, name,
version_list_add, ver_l);
if (i == 0)
{
fprintf(stderr, " ! no versions of package %s "
"found!\n", pkgspec);
return -1;
}
}
else
strlcpy(name, pkgspec, sizeof(name));
if (i == -2)
return -1;
if (i == -1)
{
fprintf(stderr, " ! find_pkg_versions(): %s\n",
strerror(errno));
return -1;
}
if (verbose > 1)
putchar('\n');
encap_listptr_reset(&lp);
while (encap_list_prev(ver_l, &lp) != 0)
{
checkver = (char *)encap_listptr_data(&lp);
encap_pkgspec_join(pkg, sizeof(pkg), name, checkver);
i = check_pkg(pkg);
}
encap_list_free(ver_l, (encap_freefunc_t)free);
return i;
}
static THREAD_RETURN netSenderMain(void *args0)
{
sender_state_t sendst = (sender_state_t) args0;
struct net_config *config = sendst->config;
struct timeval tv;
struct timespec ts;
int atEnd = 0;
int nrWaited=0;
unsigned long waitAverage=10000; /* Exponential average of last wait times */
struct slice *xmitSlice=NULL; /* slice being transmitted a first time */
struct slice *rexmitSlice=NULL; /* slice being re-transmitted */
int sliceNo = 0;
/* transmit the data */
if(config->default_slice_size == 0) {
#ifdef BB_FEATURE_UDPCAST_FEC
if(config->flags & FLAG_FEC) {
config->sliceSize =
config->fec_stripesize * config->fec_stripes;
} else
#endif
if(config->flags & FLAG_ASYNC)
config->sliceSize = 1024;
else if (sendst->config->flags & FLAG_SN) {
sendst->config->sliceSize = 112;
} else
sendst->config->sliceSize = 130;
sendst->config->discovery = DSC_DOUBLING;
} else {
config->sliceSize = config->default_slice_size;
#ifdef BB_FEATURE_UDPCAST_FEC
if((config->flags & FLAG_FEC) &&
(config->sliceSize > 128 * config->fec_stripes))
config->sliceSize = 128 * config->fec_stripes;
#endif
}
#ifdef BB_FEATURE_UDPCAST_FEC
if( (sendst->config->flags & FLAG_FEC) &&
config->max_slice_size > config->fec_stripes * 128)
config->max_slice_size = config->fec_stripes * 128;
#endif
if(config->sliceSize > config->max_slice_size)
config->sliceSize = config->max_slice_size;
assert(config->sliceSize <= MAX_SLICE_SIZE);
do {
/* first, cleanup rexmit Slice if needed */
if(rexmitSlice != NULL) {
if(rexmitSlice->nrReady ==
udpc_nrParticipants(sendst->rc.participantsDb)){
#if DEBUG
flprintf("slice is ready\n");
#endif
ackSlice(rexmitSlice, sendst->config, sendst->fifo,
sendst->stats);
}
if(isSliceAcked(rexmitSlice)) {
freeSlice(sendst, rexmitSlice);
rexmitSlice = NULL;
}
}
/* then shift xmit slice to rexmit slot, if possible */
if(rexmitSlice == NULL && xmitSlice != NULL &&
isSliceXmitted(xmitSlice)) {
rexmitSlice = xmitSlice;
xmitSlice = NULL;
sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats,
sendst->socket);
}
/* handle any messages */
if(pc_getWaiting(sendst->rc.incoming)) {
#if DEBUG
flprintf("Before message %d\n",
pc_getWaiting(sendst->rc.incoming));
#endif
handleNextMessage(sendst, xmitSlice, rexmitSlice);
/* restart at beginning of loop: we may have acked the rxmit
* slice, makeing it possible to shift the pipe */
continue;
}
/* do any needed retransmissions */
if(rexmitSlice != NULL && rexmitSlice->needRxmit) {
doRetransmissions(sendst, rexmitSlice);
/* restart at beginning: new messages may have arrived during
* retransmission */
continue;
}
/* if all participants answered, send req ack */
if(rexmitSlice != NULL &&
rexmitSlice->nrAnswered ==
udpc_nrParticipants(sendst->rc.participantsDb)) {
rexmitSlice->rxmitId++;
sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats,
sendst->socket);
}
if(xmitSlice == NULL && !atEnd) {
#if DEBUG
flprintf("SN=%d\n", sendst->config->flags & FLAG_SN);
#endif
if((sendst->config->flags & FLAG_SN) ||
rexmitSlice == NULL) {
#ifdef BB_FEATURE_UDPCAST_FEC
if(sendst->config->flags & FLAG_FEC) {
int i;
pc_consume(sendst->fec_data_pc, 1);
i = pc_getConsumerPosition(sendst->fec_data_pc);
xmitSlice = &sendst->slices[i];
pc_consumed(sendst->fec_data_pc, 1);
} else
#endif
{
xmitSlice = makeSlice(sendst, sliceNo++);
}
if(xmitSlice->bytes == 0)
atEnd = 1;
}
}
if(xmitSlice != NULL && xmitSlice->state == SLICE_NEW) {
sendSlice(sendst, xmitSlice, 0);
#if DEBUG
flprintf("%d Interrupted at %d/%d\n", xmitSlice->sliceNo,
xmitSlice->nextBlock,
getSliceBlocks(xmitSlice, sendst->config));
#endif
continue;
}
if(atEnd && rexmitSlice == NULL && xmitSlice == NULL)
break;
if(sendst->config->flags & FLAG_ASYNC)
break;
#if DEBUG
flprintf("Waiting for timeout...\n");
#endif
gettimeofday(&tv, 0);
ts.tv_sec = tv.tv_sec;
ts.tv_nsec = (tv.tv_usec + 1.1*waitAverage) * 1000;
#ifdef WINDOWS
/* Windows has a granularity of 1 millisecond in its timer. Take this
* into account here */
#define GRANULARITY 1000000
ts.tv_nsec += 3*GRANULARITY/2;
ts.tv_nsec -= ts.tv_nsec % GRANULARITY;
#endif
#define BILLION 1000000000
while(ts.tv_nsec >= BILLION) {
ts.tv_nsec -= BILLION;
ts.tv_sec++;
}
if(rexmitSlice->rxmitId > 10)
/* after tenth retransmission, wait minimum one second */
ts.tv_sec++;
if(pc_consumeAnyWithTimeout(sendst->rc.incoming, &ts) != 0) {
#if DEBUG
flprintf("Have data\n");
#endif
{
struct timeval tv2;
unsigned long timeout;
gettimeofday(&tv2, 0);
timeout =
(tv2.tv_sec - tv.tv_sec) * 1000000+
tv2.tv_usec - tv.tv_usec;
if(nrWaited)
timeout += waitAverage;
waitAverage += 9; /* compensate against rounding errors */
waitAverage = (0.9 * waitAverage + 0.1 * timeout);
}
nrWaited = 0;
continue;
}
if(rexmitSlice == NULL) {
udpc_flprintf("Weird. Timeout and no rxmit slice");
break;
}
if(nrWaited > 5){
#ifndef WINDOWS
/* on Cygwin, we would get too many of those messages... */
udpc_flprintf("Timeout notAnswered=");
udpc_printNotSet(sendst->rc.participantsDb,
rexmitSlice->answeredSet);
udpc_flprintf(" notReady=");
udpc_printNotSet(sendst->rc.participantsDb, rexmitSlice->sl_reqack.readySet);
udpc_flprintf(" nrAns=%d nrRead=%d nrPart=%d avg=%ld\n",
rexmitSlice->nrAnswered,
rexmitSlice->nrReady,
udpc_nrParticipants(sendst->rc.participantsDb),
waitAverage);
nrWaited=0;
#endif
}
nrWaited++;
if(rexmitSlice->rxmitId > config->retriesUntilDrop) {
int i;
for(i=0; i < MAX_CLIENTS; i++) {
if(udpc_isParticipantValid(sendst->rc.participantsDb, i) &&
!BIT_ISSET(i, rexmitSlice->sl_reqack.readySet)) {
udpc_flprintf("Dropping client #%d because of timeout\n",
i);
#ifdef USE_SYSLOG
syslog(LOG_INFO, "dropped client #%d because of timeout",
i);
#endif
udpc_removeParticipant(sendst->rc.participantsDb, i);
if(nrParticipants(sendst->rc.participantsDb) == 0)
exit(0);
}
}
continue;
}
rexmitSlice->rxmitId++;
sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats,
sendst->socket);
} while(udpc_nrParticipants(sendst->rc.participantsDb)||
(config->flags & FLAG_ASYNC));
cancelReturnChannel(&sendst->rc);
return 0;
}
static void
serialize_one_l_sub(FILE *f_xcl, struct xcl_l*lp, struct ilem_form *fp)
{
fputs("<l", f_xcl);
if (!fp)
x2_attr(f_xcl,"xml:id",lp->xml_id);
x2_attr_i(f_xcl,"lnum",fp ? fp->lnum : lp->lnum);
x2_attr(f_xcl,"ref",lp->ref);
x2_attr(f_xcl,"ftype",lp->subtype);
if (lp->cof_head)
{
x2_attr(f_xcl, "cof-head", lp->cof_head->xml_id);
if (fp)
fp->f2.sig = NULL;
else if (lp->f)
lp->f->f2.sig = NULL;
}
else if (lp->cof_tails)
{
struct xcl_l *tailp;
fputs(" cof-tails=\"", f_xcl);
for (tailp = list_first(lp->cof_tails); tailp; tailp = list_next(lp->cof_tails))
{
fputs(tailp->xml_id, f_xcl);
if (tailp != list_last(lp->cof_tails))
fputc(' ', f_xcl);
}
fputs("\"", f_xcl);
}
if (fp)
{
x2_attr(f_xcl,"inst",fp->sublem);
if (BIT_ISSET(fp->f2.flags,F2_FLAGS_INVALID)
|| BIT_ISSET(fp->f2.flags,F2_FLAGS_PARTIAL)
|| BIT_ISSET(fp->f2.flags,F2_FLAGS_NO_FORM))
x2_attr(f_xcl, "bad", "yes");
else
{
x2_attr(f_xcl,"sig",((const char*)fp->f2.sig));
x2_attr(f_xcl,"tail-sig",((const char*)fp->f2.tail_sig));
}
}
else
{
x2_attr(f_xcl,"inst",lp->inst);
if (lp->f)
{
if (BIT_ISSET(lp->f->f2.flags,F2_FLAGS_INVALID))
x2_attr(f_xcl, "bad", "yes");
else if (BIT_ISSET(lp->f->f2.flags, F2_FLAGS_NOT_IN_SIGS))
x2_attr(f_xcl,"newsig",(char *)lp->f->f2.sig);
else if (BIT_ISSET(lp->f->f2.flags,F2_FLAGS_PARTIAL)
|| BIT_ISSET(lp->f->f2.flags,F2_FLAGS_NO_FORM))
x2_attr(f_xcl, "bad", "yes");
else
{
if (BIT_ISSET(lp->f->f2.flags, F2_FLAGS_NEW_BY_PROJ)
|| BIT_ISSET(lp->f->f2.flags, F2_FLAGS_NEW_BY_LANG))
{
x2_attr(f_xcl,"exosig",(char *)lp->f->f2.sig);
x2_attr(f_xcl,"exoprj",(char *)lp->f->f2.exo_project);
x2_attr(f_xcl,"exolng",(char *)lp->f->f2.exo_lang);
}
else
x2_attr(f_xcl,"sig",(char *)lp->f->f2.sig);
x2_attr(f_xcl,"tail-sig",((const char*)lp->f->f2.tail_sig));
}
}
}
fputs(">",f_xcl);
if (lp->f)
{
f2_serialize_form(f_xcl, &lp->f->f2);
if (lp->f->props)
props_dump_props(lp->f,f_xcl);
}
ilem_para_dump(f_xcl, lp);
fputs("</l>",f_xcl);
}
