MessageThrowTo *
throwTo (Capability *cap, // the Capability we hold
StgTSO *source, // the TSO sending the exception (or NULL)
StgTSO *target, // the TSO receiving the exception
StgClosure *exception) // the exception closure
{
MessageThrowTo *msg;
msg = (MessageThrowTo *) allocate(cap, sizeofW(MessageThrowTo));
// the message starts locked; see below
SET_HDR(msg, &stg_WHITEHOLE_info, CCS_SYSTEM);
msg->source = source;
msg->target = target;
msg->exception = exception;
switch (throwToMsg(cap, msg))
{
case THROWTO_SUCCESS:
// unlock the message now, otherwise we leave a WHITEHOLE in
// the heap (#6103)
SET_HDR(msg, &stg_MSG_THROWTO_info, CCS_SYSTEM);
return NULL;
case THROWTO_BLOCKED:
default:
// the caller will unlock the message when it is ready. We
// cannot unlock it yet, because the calling thread will need
// to tidy up its state first.
return msg;
}
}
HaskellObj
rts_mkDouble (Capability *cap, HsDouble d)
{
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,sizeofW(StgDouble)));
SET_HDR(p, Dzh_con_info, CCS_SYSTEM);
ASSIGN_DBL((P_)p->payload, (StgDouble)d);
return p;
}
HaskellObj
rts_mkFunPtr (Capability *cap, HsFunPtr a)
{
StgClosure *p = (StgClosure *)allocate(cap,sizeofW(StgHeader)+1);
SET_HDR(p, FunPtr_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)a;
return p;
}
HaskellObj
rts_mkInt32 (Capability *cap, HsInt32 i)
{
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, I32zh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgInt)i;
return p;
}
HaskellObj
rts_mkInt64 (Capability *cap, HsInt64 i)
{
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,2));
SET_HDR(p, I64zh_con_info, CCS_SYSTEM);
ASSIGN_Int64((P_)&(p->payload[0]), i);
return p;
}
HaskellObj
rts_mkWord (Capability *cap, HsWord i)
{
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, Wzh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgWord)i;
return p;
}
HaskellObj
rts_mkFloat (Capability *cap, HsFloat f)
{
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, Fzh_con_info, CCS_SYSTEM);
ASSIGN_FLT((P_)p->payload, (StgFloat)f);
return p;
}
HaskellObj
rts_mkWord64 (Capability *cap, HsWord64 w)
{
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,2));
/* see mk_Int8 comment */
SET_HDR(p, W64zh_con_info, CCS_SYSTEM);
ASSIGN_Word64((P_)&(p->payload[0]), w);
return p;
}
HaskellObj
rts_mkWord8 (Capability *cap, HsWord8 w)
{
/* see rts_mkInt* comments */
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, W8zh_con_info, CCS_SYSTEM);
p->payload[0] = (StgClosure *)(StgWord)(w & 0xff);
return p;
}
HaskellObj
rts_mkInt16 (Capability *cap, HsInt16 i)
{
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, I16zh_con_info, CCS_SYSTEM);
/* Make sure we mask out the relevant bits */
p->payload[0] = (StgClosure *)(StgInt)i;
return p;
}
HaskellObj
rts_mkInt8 (Capability *cap, HsInt8 i)
{
StgClosure *p = (StgClosure *)allocate(cap,CONSTR_sizeW(0,1));
SET_HDR(p, I8zh_con_info, CCS_SYSTEM);
/* Make sure we mask out the bits above the lowest 8 */
p->payload[0] = (StgClosure *)(StgInt)i;
return p;
}
STATIC_INLINE StgInd *
lockCAF (StgRegTable *reg, StgIndStatic *caf)
{
const StgInfoTable *orig_info;
Capability *cap = regTableToCapability(reg);
StgInd *bh;
orig_info = caf->header.info;
#ifdef THREADED_RTS
const StgInfoTable *cur_info;
if (orig_info == &stg_IND_STATIC_info ||
orig_info == &stg_WHITEHOLE_info) {
// already claimed by another thread; re-enter the CAF
return NULL;
}
cur_info = (const StgInfoTable *)
cas((StgVolatilePtr)&caf->header.info,
(StgWord)orig_info,
(StgWord)&stg_WHITEHOLE_info);
if (cur_info != orig_info) {
// already claimed by another thread; re-enter the CAF
return NULL;
}
// successfully claimed by us; overwrite with IND_STATIC
#endif
// For the benefit of revertCAFs(), save the original info pointer
caf->saved_info = orig_info;
// Allocate the blackhole indirection closure
bh = (StgInd *)allocate(cap, sizeofW(*bh));
SET_HDR(bh, &stg_CAF_BLACKHOLE_info, caf->header.prof.ccs);
bh->indirectee = (StgClosure *)cap->r.rCurrentTSO;
caf->indirectee = (StgClosure *)bh;
write_barrier();
SET_INFO((StgClosure*)caf,&stg_IND_STATIC_info);
return bh;
}
StgCompactNFData *
compactNew (Capability *cap, StgWord size)
{
StgWord aligned_size;
StgCompactNFDataBlock *block;
StgCompactNFData *self;
bdescr *bd;
aligned_size = BLOCK_ROUND_UP(size + sizeof(StgCompactNFData)
+ sizeof(StgCompactNFDataBlock));
// Don't allow sizes larger than a megablock, because we can't use the
// memory after the first mblock for storing objects.
if (aligned_size >= BLOCK_SIZE * BLOCKS_PER_MBLOCK)
aligned_size = BLOCK_SIZE * BLOCKS_PER_MBLOCK;
block = compactAllocateBlockInternal(cap, aligned_size, NULL,
ALLOCATE_NEW);
self = firstBlockGetCompact(block);
SET_HDR((StgClosure*)self, &stg_COMPACT_NFDATA_CLEAN_info, CCS_SYSTEM);
self->autoBlockW = aligned_size / sizeof(StgWord);
self->nursery = block;
self->last = block;
self->hash = NULL;
block->owner = self;
bd = Bdescr((P_)block);
bd->free = (StgPtr)((W_)self + sizeof(StgCompactNFData));
self->hp = bd->free;
self->hpLim = bd->start + bd->blocks * BLOCK_SIZE_W;
self->totalW = bd->blocks * BLOCK_SIZE_W;
debugTrace(DEBUG_compact, "compactNew: size %" FMT_Word, size);
return self;
}
StgMutArrPtrs *gtc_heap_view_closurePtrs(Capability *cap, StgClosure *closure) {
ASSERT(LOOKS_LIKE_CLOSURE_PTR(closure));
StgWord size = gtc_heap_view_closureSize(closure);
StgWord nptrs = 0;
StgWord i;
// First collect all pointers here, with the comfortable memory bound
// of the whole closure. Afterwards we know how many pointers are in
// the closure and then we can allocate space on the heap and copy them
// there
StgClosure *ptrs[size];
StgClosure **end;
StgClosure **ptr;
StgInfoTable *info = get_itbl(closure);
StgThunkInfoTable *thunk_info;
StgFunInfoTable *fun_info;
switch (info->type) {
case INVALID_OBJECT:
barf("Invalid Object");
break;
// No pointers
case ARR_WORDS:
break;
// Default layout
case CONSTR_1_0:
case CONSTR_0_1:
case CONSTR_2_0:
case CONSTR_1_1:
case CONSTR_0_2:
case CONSTR:
case CONSTR_STATIC:
case CONSTR_NOCAF_STATIC:
case PRIM:
case FUN:
case FUN_1_0:
case FUN_0_1:
case FUN_1_1:
case FUN_2_0:
case FUN_0_2:
case FUN_STATIC:
end = closure->payload + info->layout.payload.ptrs;
for (ptr = closure->payload; ptr < end; ptr++) {
ptrs[nptrs++] = *ptr;
}
break;
case THUNK:
case THUNK_1_0:
case THUNK_0_1:
case THUNK_1_1:
case THUNK_2_0:
case THUNK_0_2:
case THUNK_STATIC:
end = ((StgThunk *)closure)->payload + info->layout.payload.ptrs;
for (ptr = ((StgThunk *)closure)->payload; ptr < end; ptr++) {
ptrs[nptrs++] = *ptr;
}
break;
case THUNK_SELECTOR:
ptrs[nptrs++] = ((StgSelector *)closure)->selectee;
break;
case AP:
ptrs[nptrs++] = ((StgAP *)closure)->fun;
gtc_heap_view_closure_ptrs_in_pap_payload(ptrs, &nptrs,
((StgAP *)closure)->fun,
((StgAP *)closure)->payload,
((StgAP *)closure)->n_args);
break;
case PAP:
ptrs[nptrs++] = ((StgPAP *)closure)->fun;
gtc_heap_view_closure_ptrs_in_pap_payload(ptrs, &nptrs,
((StgPAP *)closure)->fun,
((StgPAP *)closure)->payload,
((StgPAP *)closure)->n_args);
break;
case AP_STACK:
ptrs[nptrs++] = ((StgAP_STACK *)closure)->fun;
for (i = 0; i < ((StgAP_STACK *)closure)->size; ++i) {
ptrs[nptrs++] = ((StgAP_STACK *)closure)->payload[i];
}
break;
case BCO:
ptrs[nptrs++] = (StgClosure *)((StgBCO *)closure)->instrs;
ptrs[nptrs++] = (StgClosure *)((StgBCO *)closure)->literals;
ptrs[nptrs++] = (StgClosure *)((StgBCO *)closure)->ptrs;
break;
case IND:
case IND_PERM:
case IND_STATIC:
case BLACKHOLE:
ptrs[nptrs++] = (StgClosure *)(((StgInd *)closure)->indirectee);
break;
case MUT_ARR_PTRS_CLEAN:
case MUT_ARR_PTRS_DIRTY:
case MUT_ARR_PTRS_FROZEN:
case MUT_ARR_PTRS_FROZEN0:
for (i = 0; i < ((StgMutArrPtrs *)closure)->ptrs; ++i) {
ptrs[nptrs++] = ((StgMutArrPtrs *)closure)->payload[i];
}
break;
case MUT_VAR_CLEAN:
ptrs[nptrs++] = ((StgMutVar *)closure)->var;
break;
case MVAR_DIRTY:
case MVAR_CLEAN:
ptrs[nptrs++] = (StgClosure *)((StgMVar *)closure)->head;
ptrs[nptrs++] = (StgClosure *)((StgMVar *)closure)->tail;
ptrs[nptrs++] = ((StgMVar *)closure)->value;
break;
default:
//fprintf(stderr,"closurePtrs: Cannot handle type %s yet\n", gtc_heap_view_closure_type_names[info->type]);
break;
}
size = nptrs + mutArrPtrsCardTableSize(nptrs);
StgMutArrPtrs *arr =
(StgMutArrPtrs *)allocate(cap, sizeofW(StgMutArrPtrs) + size);
TICK_ALLOC_PRIM(sizeofW(StgMutArrPtrs), nptrs, 0);
SET_HDR(arr, &stg_MUT_ARR_PTRS_FROZEN_info, CCCS);
arr->ptrs = nptrs;
arr->size = size;
for (i = 0; i<nptrs; i++) {
arr->payload[i] = ptrs[i];
}
return arr;
}
void
tryWakeupThread (Capability *cap, StgTSO *tso)
{
traceEventThreadWakeup (cap, tso, tso->cap->no);
#ifdef THREADED_RTS
if (tso->cap != cap)
{
MessageWakeup *msg;
msg = (MessageWakeup *)allocate(cap,sizeofW(MessageWakeup));
SET_HDR(msg, &stg_MSG_TRY_WAKEUP_info, CCS_SYSTEM);
msg->tso = tso;
sendMessage(cap, tso->cap, (Message*)msg);
debugTraceCap(DEBUG_sched, cap, "message: try wakeup thread %ld on cap %d",
(W_)tso->id, tso->cap->no);
return;
}
#endif
switch (tso->why_blocked)
{
case BlockedOnMVar:
case BlockedOnMVarRead:
{
if (tso->_link == END_TSO_QUEUE) {
tso->block_info.closure = (StgClosure*)END_TSO_QUEUE;
goto unblock;
} else {
return;
}
}
case BlockedOnMsgThrowTo:
{
const StgInfoTable *i;
i = lockClosure(tso->block_info.closure);
unlockClosure(tso->block_info.closure, i);
if (i != &stg_MSG_NULL_info) {
debugTraceCap(DEBUG_sched, cap, "thread %ld still blocked on throwto (%p)",
(W_)tso->id, tso->block_info.throwto->header.info);
return;
}
// remove the block frame from the stack
ASSERT(tso->stackobj->sp[0] == (StgWord)&stg_block_throwto_info);
tso->stackobj->sp += 3;
goto unblock;
}
case BlockedOnBlackHole:
case BlockedOnSTM:
case ThreadMigrating:
goto unblock;
default:
// otherwise, do nothing
return;
}
unblock:
// just run the thread now, if the BH is not really available,
// we'll block again.
tso->why_blocked = NotBlocked;
appendToRunQueue(cap,tso);
// We used to set the context switch flag here, which would
// trigger a context switch a short time in the future (at the end
// of the current nursery block). The idea is that we have just
// woken up a thread, so we may need to load-balance and migrate
// threads to other CPUs. On the other hand, setting the context
// switch flag here unfairly penalises the current thread by
// yielding its time slice too early.
//
// The synthetic benchmark nofib/smp/chan can be used to show the
// difference quite clearly.
// cap->context_switch = 1;
}
/* ---------------------------------------------------------------------------
Create a new thread.
The new thread starts with the given stack size. Before the
scheduler can run, however, this thread needs to have a closure
(and possibly some arguments) pushed on its stack. See
pushClosure() in Schedule.h.
createGenThread() and createIOThread() (in SchedAPI.h) are
convenient packaged versions of this function.
------------------------------------------------------------------------ */
StgTSO *
createThread(Capability *cap, W_ size)
{
StgTSO *tso;
StgStack *stack;
nat stack_size;
/* sched_mutex is *not* required */
/* catch ridiculously small stack sizes */
if (size < MIN_STACK_WORDS + sizeofW(StgStack) + sizeofW(StgTSO)) {
size = MIN_STACK_WORDS + sizeofW(StgStack) + sizeofW(StgTSO);
}
/* The size argument we are given includes all the per-thread
* overheads:
*
* - The TSO structure
* - The STACK header
*
* This is so that we can use a nice round power of 2 for the
* default stack size (e.g. 1k), and if we're allocating lots of
* threads back-to-back they'll fit nicely in a block. It's a bit
* of a benchmark hack, but it doesn't do any harm.
*/
stack_size = round_to_mblocks(size - sizeofW(StgTSO));
stack = (StgStack *)allocate(cap, stack_size);
TICK_ALLOC_STACK(stack_size);
SET_HDR(stack, &stg_STACK_info, cap->r.rCCCS);
stack->stack_size = stack_size - sizeofW(StgStack);
stack->sp = stack->stack + stack->stack_size;
stack->dirty = 1;
tso = (StgTSO *)allocate(cap, sizeofW(StgTSO));
TICK_ALLOC_TSO();
SET_HDR(tso, &stg_TSO_info, CCS_SYSTEM);
// Always start with the compiled code evaluator
tso->what_next = ThreadRunGHC;
tso->why_blocked = NotBlocked;
tso->block_info.closure = (StgClosure *)END_TSO_QUEUE;
tso->blocked_exceptions = END_BLOCKED_EXCEPTIONS_QUEUE;
tso->bq = (StgBlockingQueue *)END_TSO_QUEUE;
tso->flags = 0;
tso->dirty = 1;
tso->_link = END_TSO_QUEUE;
tso->saved_errno = 0;
tso->bound = NULL;
tso->cap = cap;
tso->stackobj = stack;
tso->tot_stack_size = stack->stack_size;
ASSIGN_Int64((W_*)&(tso->alloc_limit), 0);
tso->trec = NO_TREC;
#ifdef PROFILING
tso->prof.cccs = CCS_MAIN;
#endif
// put a stop frame on the stack
stack->sp -= sizeofW(StgStopFrame);
SET_HDR((StgClosure*)stack->sp,
(StgInfoTable *)&stg_stop_thread_info,CCS_SYSTEM);
/* Link the new thread on the global thread list.
*/
ACQUIRE_LOCK(&sched_mutex);
tso->id = next_thread_id++; // while we have the mutex
tso->global_link = g0->threads;
g0->threads = tso;
RELEASE_LOCK(&sched_mutex);
// ToDo: report the stack size in the event?
traceEventCreateThread(cap, tso);
return tso;
}
nat messageBlackHole(Capability *cap, MessageBlackHole *msg)
{
const StgInfoTable *info;
StgClosure *p;
StgBlockingQueue *bq;
StgClosure *bh = UNTAG_CLOSURE(msg->bh);
StgTSO *owner;
debugTraceCap(DEBUG_sched, cap, "message: thread %d blocking on blackhole %p",
(lnat)msg->tso->id, msg->bh);
info = bh->header.info;
// If we got this message in our inbox, it might be that the
// BLACKHOLE has already been updated, and GC has shorted out the
// indirection, so the pointer no longer points to a BLACKHOLE at
// all.
if (info != &stg_BLACKHOLE_info &&
info != &stg_CAF_BLACKHOLE_info &&
info != &__stg_EAGER_BLACKHOLE_info &&
info != &stg_WHITEHOLE_info) {
// if it is a WHITEHOLE, then a thread is in the process of
// trying to BLACKHOLE it. But we know that it was once a
// BLACKHOLE, so there is at least a valid pointer in the
// payload, so we can carry on.
return 0;
}
// The blackhole must indirect to a TSO, a BLOCKING_QUEUE, an IND,
// or a value.
loop:
// NB. VOLATILE_LOAD(), because otherwise gcc hoists the load
// and turns this into an infinite loop.
p = UNTAG_CLOSURE((StgClosure*)VOLATILE_LOAD(&((StgInd*)bh)->indirectee));
info = p->header.info;
if (info == &stg_IND_info)
{
// This could happen, if e.g. we got a BLOCKING_QUEUE that has
// just been replaced with an IND by another thread in
// updateThunk(). In which case, if we read the indirectee
// again we should get the value.
goto loop;
}
else if (info == &stg_TSO_info)
{
owner = (StgTSO*)p;
#ifdef THREADED_RTS
if (owner->cap != cap) {
sendMessage(cap, owner->cap, (Message*)msg);
debugTraceCap(DEBUG_sched, cap, "forwarding message to cap %d", owner->cap->no);
return 1;
}
#endif
// owner is the owner of the BLACKHOLE, and resides on this
// Capability. msg->tso is the first thread to block on this
// BLACKHOLE, so we first create a BLOCKING_QUEUE object.
bq = (StgBlockingQueue*)allocate(cap, sizeofW(StgBlockingQueue));
// initialise the BLOCKING_QUEUE object
SET_HDR(bq, &stg_BLOCKING_QUEUE_DIRTY_info, CCS_SYSTEM);
bq->bh = bh;
bq->queue = msg;
bq->owner = owner;
msg->link = (MessageBlackHole*)END_TSO_QUEUE;
// All BLOCKING_QUEUES are linked in a list on owner->bq, so
// that we can search through them in the event that there is
// a collision to update a BLACKHOLE and a BLOCKING_QUEUE
// becomes orphaned (see updateThunk()).
bq->link = owner->bq;
owner->bq = bq;
dirty_TSO(cap, owner); // we modified owner->bq
// If the owner of the blackhole is currently runnable, then
// bump it to the front of the run queue. This gives the
// blocked-on thread a little boost which should help unblock
// this thread, and may avoid a pile-up of other threads
// becoming blocked on the same BLACKHOLE (#3838).
//
// NB. we check to make sure that the owner is not the same as
// the current thread, since in that case it will not be on
// the run queue.
if (owner->why_blocked == NotBlocked && owner->id != msg->tso->id) {
removeFromRunQueue(cap, owner);
pushOnRunQueue(cap,owner);
}
// point to the BLOCKING_QUEUE from the BLACKHOLE
write_barrier(); // make the BQ visible
((StgInd*)bh)->indirectee = (StgClosure *)bq;
recordClosureMutated(cap,bh); // bh was mutated
debugTraceCap(DEBUG_sched, cap, "thread %d blocked on thread %d",
(lnat)msg->tso->id, (lnat)owner->id);
return 1; // blocked
}
else if (info == &stg_BLOCKING_QUEUE_CLEAN_info ||
info == &stg_BLOCKING_QUEUE_DIRTY_info)
{
StgBlockingQueue *bq = (StgBlockingQueue *)p;
ASSERT(bq->bh == bh);
owner = bq->owner;
ASSERT(owner != END_TSO_QUEUE);
#ifdef THREADED_RTS
if (owner->cap != cap) {
sendMessage(cap, owner->cap, (Message*)msg);
debugTraceCap(DEBUG_sched, cap, "forwarding message to cap %d", owner->cap->no);
return 1;
}
#endif
msg->link = bq->queue;
bq->queue = msg;
recordClosureMutated(cap,(StgClosure*)msg);
if (info == &stg_BLOCKING_QUEUE_CLEAN_info) {
bq->header.info = &stg_BLOCKING_QUEUE_DIRTY_info;
recordClosureMutated(cap,(StgClosure*)bq);
}
debugTraceCap(DEBUG_sched, cap, "thread %d blocked on thread %d",
(lnat)msg->tso->id, (lnat)owner->id);
// See above, #3838
if (owner->why_blocked == NotBlocked && owner->id != msg->tso->id) {
removeFromRunQueue(cap, owner);
pushOnRunQueue(cap,owner);
}
return 1; // blocked
}
return 0; // not blocked
}
int main(int argc, char** argv) {
s32 opt;
u32 loop_cnt = 0, purge_age = 0, seed;
u8 sig_loaded = 0, show_once = 0, no_statistics = 0,
display_mode = 0, has_fake = 0;
s32 oindex = 0;
u8 *wordlist = NULL, *output_dir = NULL;
u8 *sig_list_strg = NULL;
u8 *gtimeout_str = NULL;
u32 gtimeout = 0;
struct termios term;
struct timeval tv;
u64 st_time, en_time;
signal(SIGINT, ctrlc_handler);
signal(SIGWINCH, resize_handler);
signal(SIGPIPE, SIG_IGN);
SSL_library_init();
/* Options, options, and options */
static struct option long_options[] = {
{"auth", required_argument, 0, 'A' },
{"host", required_argument, 0, 'F' },
{"cookie", required_argument, 0, 'C' },
{"reject-cookies", required_argument, 0, 'N' },
{"header", required_argument, 0, 'H' },
{"user-agent", required_argument, 0, 'b' },
#ifdef PROXY_SUPPORT
{"proxy", required_argument, 0, 'J' },
#endif /* PROXY_SUPPORT */
{"max-depth", required_argument, 0, 'd' },
{"max-child", required_argument, 0, 'c' },
{"max-descendants", required_argument, 0, 'x' },
{"max-requests", required_argument, 0, 'r' },
{"max-rate", required_argument, 0, 'l'},
{"probability", required_argument, 0, 'p' },
{"seed", required_argument, 0, 'q' },
{"include", required_argument, 0, 'I' },
{"exclude", required_argument, 0, 'X' },
{"skip-param", required_argument, 0, 'K' },
{"skip-forms", no_argument, 0, 'O' },
{"include-domain", required_argument, 0, 'D' },
{"ignore-links", no_argument, 0, 'P' },
{"no-ext-fuzzing", no_argument, 0, 'Y' },
{"log-mixed-content", no_argument, 0, 'M' },
{"skip-error-pages", no_argument, 0, 'Z' },
{"log-external-urls", no_argument, 0, 'U' },
{"log-cache-mismatches", no_argument, 0, 'E' },
{"form-value", no_argument, 0, 'T' },
{"rw-wordlist", required_argument, 0, 'W' },
{"no-keyword-learning", no_argument, 0, 'L' },
{"mode", required_argument, 0, 'J' },
{"wordlist", required_argument, 0, 'S'},
{"trust-domain", required_argument, 0, 'B' },
{"max-connections", required_argument, 0, 'g' },
{"max-host-connections", required_argument, 0, 'm' },
{"max-fail", required_argument, 0, 'f' },
{"request-timeout", required_argument, 0, 't' },
{"network-timeout", required_argument, 0, 'w' },
{"idle-timeout", required_argument, 0, 'i' },
{"response-size", required_argument, 0, 's' },
{"discard-binary", required_argument, 0, 'e' },
{"output", required_argument, 0, 'o' },
{"help", no_argument, 0, 'h' },
{"quiet", no_argument, 0, 'u' },
{"verbose", no_argument, 0, 'v' },
{"scan-timeout", required_argument, 0, 'k'},
{"signatures", required_argument, 0, 'z'},
{"checks", no_argument, 0, 0},
{"checks-toggle", required_argument, 0, 0},
{"no-checks", no_argument, 0, 0},
{"fast", no_argument, 0, 0},
{"auth-form", required_argument, 0, 0},
{"auth-form-target", required_argument, 0, 0},
{"auth-user", required_argument, 0, 0},
{"auth-user-field", required_argument, 0, 0},
{"auth-pass", required_argument, 0, 0},
{"auth-pass-field", required_argument, 0, 0},
{"auth-verify-url", required_argument, 0, 0},
{0, 0, 0, 0 }
};
/* Come up with a quasi-decent random seed. */
gettimeofday(&tv, NULL);
seed = tv.tv_usec ^ (tv.tv_sec << 16) ^ getpid();
SAY("skipfish version " VERSION " by <*****@*****.**>\n");
while ((opt = getopt_long(argc, argv,
"+A:B:C:D:EF:G:H:I:J:K:LMNOPQR:S:T:UW:X:YZ"
"b:c:d:ef:g:hi:k:l:m:o:p:q:r:s:t:uvw:x:z:",
long_options, &oindex)) >= 0)
switch (opt) {
case 'A': {
u8* x = (u8*)strchr(optarg, ':');
if (!x) FATAL("Credentials must be in 'user:pass' form.");
*(x++) = 0;
auth_user = (u8*)optarg;
auth_pass = x;
auth_type = AUTH_BASIC;
break;
}
#ifdef PROXY_SUPPORT
case 'J': {
u8* x = (u8*)strchr(optarg, ':');
if (!x) FATAL("Proxy data must be in 'host:port' form.");
*(x++) = 0;
use_proxy = (u8*)optarg;
use_proxy_port = atoi((char*)x);
if (!use_proxy_port) FATAL("Incorrect proxy port number.");
break;
}
#endif /* PROXY_SUPPORT */
case 'F': {
u8* x = (u8*)strchr(optarg, '=');
u32 fake_addr;
if (!x) FATAL("Fake mappings must be in 'host=IP' form.");
*x = 0;
fake_addr = inet_addr((char*)x + 1);
if (fake_addr == (u32)-1)
FATAL("Could not parse IP address '%s'.", x + 1);
fake_host((u8*)optarg, fake_addr);
has_fake = 1;
break;
}
case 'H': {
u8* x = (u8*)strchr(optarg, '=');
if (!x) FATAL("Extra headers must be in 'name=value' form.");
*x = 0;
if (!strcasecmp(optarg, "Cookie"))
FATAL("Do not use -H to set cookies (try -C instead).");
SET_HDR((u8*)optarg, x + 1, &global_http_par);
break;
}
case 'C': {
u8* x = (u8*)strchr(optarg, '=');
if (!x) FATAL("Cookies must be in 'name=value' form.");
if (strchr(optarg, ';'))
FATAL("Split multiple cookies into separate -C options.");
*x = 0;
SET_CK((u8*)optarg, x + 1, &global_http_par);
break;
}
case 'D':
if (*optarg == '*') optarg++;
APPEND_FILTER(allow_domains, num_allow_domains, optarg);
break;
case 'K':
APPEND_FILTER(skip_params, num_skip_params, optarg);
break;
case 'B':
if (*optarg == '*') optarg++;
APPEND_FILTER(trust_domains, num_trust_domains, optarg);
break;
case 'I':
if (*optarg == '*') optarg++;
APPEND_FILTER(allow_urls, num_allow_urls, optarg);
break;
case 'X':
if (*optarg == '*') optarg++;
APPEND_FILTER(deny_urls, num_deny_urls, optarg);
break;
case 'T': {
u8* x = (u8*)strchr(optarg, '=');
if (!x) FATAL("Rules must be in 'name=value' form.");
*x = 0;
add_form_hint((u8*)optarg, x + 1);
break;
}
case 'N':
ignore_cookies = 1;
break;
case 'Y':
no_fuzz_ext = 1;
break;
case 'q':
if (sscanf(optarg, "0x%08x", &seed) != 1)
FATAL("Invalid seed format.");
srandom(seed);
break;
case 'Q':
suppress_dupes = 1;
break;
case 'P':
no_parse = 1;
break;
case 'M':
warn_mixed = 1;
break;
case 'U':
log_ext_urls = 1;
break;
case 'L':
dont_add_words = 1;
break;
case 'E':
pedantic_cache = 1;
break;
case 'O':
no_forms = 1;
break;
case 'R':
purge_age = atoi(optarg);
if (purge_age < 3) FATAL("Purge age invalid or too low (min 3).");
break;
case 'd':
max_depth = atoi(optarg);
if (max_depth < 2) FATAL("Invalid value '%s'.", optarg);
break;
case 'c':
max_children = atoi(optarg);
if (!max_children) FATAL("Invalid value '%s'.", optarg);
break;
case 'x':
max_descendants = atoi(optarg);
if (!max_descendants) FATAL("Invalid value '%s'.", optarg);
break;
case 'p':
crawl_prob = atoi(optarg);
if (!crawl_prob) FATAL("Invalid value '%s'.", optarg);
break;
case 'W':
if (wordlist)
FATAL("Only one -W parameter permitted (use -S to load supplemental dictionaries).");
if (!strcmp(optarg, "-")) wordlist = (u8*)"/dev/null";
else wordlist = (u8*)optarg;
break;
case 'S':
load_keywords((u8*)optarg, 1, 0);
break;
case 'z':
load_signatures((u8*)optarg);
sig_loaded = 1;
break;
case 'b':
if (optarg[0] == 'i') browser_type = BROWSER_MSIE; else
if (optarg[0] == 'f') browser_type = BROWSER_FFOX; else
if (optarg[0] == 'p') browser_type = BROWSER_PHONE; else
usage(argv[0]);
break;
case 'g':
max_connections = atoi(optarg);
if (!max_connections) FATAL("Invalid value '%s'.", optarg);
break;
case 'm':
max_conn_host = atoi(optarg);
if (!max_conn_host) FATAL("Invalid value '%s'.", optarg);
break;
case 'G':
max_guesses = atoi(optarg);
if (!max_guesses) FATAL("Invalid value '%s'.", optarg);
break;
case 'r':
max_requests = atoi(optarg);
if (!max_requests) FATAL("Invalid value '%s'.", optarg);
break;
case 'l':
max_requests_sec = atof(optarg);
if (!max_requests_sec) FATAL("Invalid value '%s'.", optarg);
break;
case 'f':
max_fail = atoi(optarg);
if (!max_fail) FATAL("Invalid value '%s'.", optarg);
break;
case 't':
resp_tmout = atoi(optarg);
if (!resp_tmout) FATAL("Invalid value '%s'.", optarg);
break;
case 'w':
rw_tmout = atoi(optarg);
if (!rw_tmout) FATAL("Invalid value '%s'.", optarg);
break;
case 'i':
idle_tmout = atoi(optarg);
if (!idle_tmout) FATAL("Invalid value '%s'.", optarg);
break;
case 's':
size_limit = atoi(optarg);
if (!size_limit) FATAL("Invalid value '%s'.", optarg);
break;
case 'o':
if (output_dir) FATAL("Multiple -o options not allowed.");
output_dir = (u8*)optarg;
rmdir(optarg);
if (mkdir(optarg, 0755))
PFATAL("Unable to create '%s'.", output_dir);
break;
case 'u':
no_statistics = 1;
break;
case 'v':
verbosity++;
break;
case 'e':
delete_bin = 1;
break;
case 'k':
if (gtimeout_str) FATAL("Multiple -k options not allowed.");
gtimeout_str = (u8*)optarg;
break;
case 'Z':
no_500_dir = 1;
break;
case 0:
if (!strcmp("checks", long_options[oindex].name ))
display_injection_checks();
if (!strcmp("checks-toggle", long_options[oindex].name ))
toggle_injection_checks((u8*)optarg, 1, 1);
if (!strcmp("no-checks", long_options[oindex].name ))
no_checks = 1;
if (!strcmp("signatures", long_options[oindex].name ))
load_signatures((u8*)optarg);
if(!strcmp("fast", long_options[oindex].name ))
toggle_injection_checks((u8*)"2,4,5,13,14,15,16", 0, 0);
if (!strcmp("auth-form", long_options[oindex].name ))
auth_form = (u8*)optarg;
if (!strcmp("auth-user", long_options[oindex].name ))
auth_user = (u8*)optarg;
if (!strcmp("auth-pass", long_options[oindex].name ))
auth_pass = (u8*)optarg;
if (!strcmp("auth-pass-field", long_options[oindex].name ))
auth_pass_field = (u8*)optarg;
if (!strcmp("auth-user-field", long_options[oindex].name ))
auth_user_field = (u8*)optarg;
if (!strcmp("auth-form-target", long_options[oindex].name ))
auth_form_target = (u8*)optarg;
if (!strcmp("auth-verify-url", long_options[oindex].name ))
auth_verify_url = (u8*)optarg;
break;
default:
usage(argv[0]);
}
#ifdef PROXY_SUPPORT
if (has_fake && use_proxy)
FATAL("-F and -J should not be used together.");
#endif /* PROXY_SUPPORT */
if (access(ASSETS_DIR "/index.html", R_OK))
PFATAL("Unable to access '%s/index.html' - wrong directory?", ASSETS_DIR);
srandom(seed);
if (optind == argc)
FATAL("Scan target not specified (try -h for help).");
if (!output_dir)
FATAL("Output directory not specified (try -h for help).");
if(verbosity && !no_statistics && isatty(2))
FATAL("Please use -v in combination with the -u flag or, "
"run skipfish while redirecting stderr to a file. ");
if (resp_tmout < rw_tmout)
resp_tmout = rw_tmout;
if (max_connections < max_conn_host)
max_connections = max_conn_host;
/* Parse the timeout string - format h:m:s */
if (gtimeout_str) {
int i = 0;
int m[3] = { 1, 60, 3600 };
u8* tok = (u8*)strtok((char*)gtimeout_str, ":");
while(tok && i <= 2) {
gtimeout += atoi((char*)tok) * m[i];
tok = (u8*)strtok(NULL, ":");
i++;
}
if(!gtimeout)
FATAL("Wrong timeout format, please use h:m:s (hours, minutes, seconds)");
DEBUG("* Scan timeout is set to %d seconds\n", gtimeout);
}
if (!wordlist) {
wordlist = (u8*)"/dev/null";
DEBUG("* No wordlist specified with -W: defaulting to /dev/null\n");
}
/* If no signature files have been specified via command-line: load
the default file */
if (!sig_loaded)
load_signatures((u8*)SIG_FILE);
load_keywords(wordlist, 0, purge_age);
/* Load the signatures list for the matching */
if (sig_list_strg) load_signatures(sig_list_strg);
/* Try to authenticate when the auth_user and auth_pass fields are set. */
if (auth_user && auth_pass) {
authenticate();
while (next_from_queue()) {
usleep(1000);
}
switch (auth_state) {
case ASTATE_DONE:
DEBUGC(L1, "*- Authentication succeeded!\n");
break;
default:
DEBUG("Auth state: %d\n", auth_state);
FATAL("Authentication failed (use -uv for more info)\n");
break;
}
}
/* Schedule all URLs in the command line for scanning. */
while (optind < argc) {
struct http_request *req;
/* Support @ notation for reading URL lists from files. */
if (argv[optind][0] == '@') {
read_urls((u8*)argv[optind++] + 1);
continue;
}
req = ck_alloc(sizeof(struct http_request));
if (parse_url((u8*)argv[optind], req, NULL))
FATAL("Scan target '%s' is not a valid absolute URL.", argv[optind]);
if (!url_allowed_host(req))
APPEND_FILTER(allow_domains, num_allow_domains,
__DFL_ck_strdup(req->host));
if (!url_allowed(req))
FATAL("URL '%s' explicitly excluded by -I / -X rules.", argv[optind]);
maybe_add_pivot(req, NULL, 2);
destroy_request(req);
optind++;
}
/* Char-by char stdin. */
tcgetattr(0, &term);
term.c_lflag &= ~ICANON;
tcsetattr(0, TCSANOW, &term);
fcntl(0, F_SETFL, O_NONBLOCK);
gettimeofday(&tv, NULL);
st_time = tv.tv_sec * 1000LL + tv.tv_usec / 1000;
#ifdef SHOW_SPLASH
if (!no_statistics) splash_screen();
#endif /* SHOW_SPLASH */
if (!no_statistics) SAY("\x1b[H\x1b[J");
else SAY(cLGN "[*] " cBRI "Scan in progress, please stay tuned...\n");
u64 refresh_time = 0;
/* Enter the crawler loop */
while ((next_from_queue() && !stop_soon) || (!show_once++)) {
u8 keybuf[8];
u64 end_time;
u64 run_time;
struct timeval tv_tmp;
gettimeofday(&tv_tmp, NULL);
end_time = tv_tmp.tv_sec * 1000LL + tv_tmp.tv_usec / 1000;
run_time = end_time - st_time;
if (gtimeout > 0 && run_time && run_time/1000 > gtimeout) {
DEBUG("* Stopping scan due to timeout\n");
stop_soon = 1;
}
req_sec = (req_count - queue_cur / 1.15) * 1000 / (run_time + 1);
if (no_statistics || ((loop_cnt++ % 100) && !show_once && idle == 0))
continue;
if (end_time > refresh_time) {
refresh_time = (end_time + 10);
if (clear_screen) {
SAY("\x1b[H\x1b[2J");
clear_screen = 0;
}
SAY(cYEL "\x1b[H"
"skipfish version " VERSION " by [email protected]\n\n"
cBRI " -" cPIN " %s " cBRI "-\n\n" cNOR,
allow_domains[0]);
if (!display_mode) {
http_stats(st_time);
SAY("\n");
database_stats();
} else {
http_req_list();
}
SAY(" \r");
}
if (fread(keybuf, 1, sizeof(keybuf), stdin) > 0) {
display_mode ^= 1;
clear_screen = 1;
}
}
gettimeofday(&tv, NULL);
en_time = tv.tv_sec * 1000LL + tv.tv_usec / 1000;
SAY("\n");
if (stop_soon)
SAY(cYEL "[!] " cBRI "Scan aborted by user, bailing out!" cNOR "\n");
term.c_lflag |= ICANON;
tcsetattr(0, TCSANOW, &term);
fcntl(0, F_SETFL, O_SYNC);
save_keywords((u8*)wordlist);
write_report(output_dir, en_time - st_time, seed);
#ifdef LOG_STDERR
SAY("\n== PIVOT DEBUG ==\n");
dump_pivots(0, 0);
SAY("\n== END OF DUMP ==\n\n");
#endif /* LOG_STDERR */
SAY(cLGN "[+] " cBRI "This was a great day for science!" cRST "\n\n");
#ifdef DEBUG_ALLOCATOR
if (!stop_soon) {
destroy_database();
destroy_signature_lists();
destroy_http();
destroy_signatures();
__TRK_report();
}
#endif /* DEBUG_ALLOCATOR */
fflush(0);
EVP_cleanup();
CRYPTO_cleanup_all_ex_data();
return 0;
}
void
scheduleFinalizers(Capability *cap, StgWeak *list)
{
StgWeak *w;
StgTSO *t;
StgMutArrPtrs *arr;
StgWord size;
nat n, i;
Task *task;
task = myTask();
if (task != NULL) {
task->running_finalizers = rtsTrue;
}
// count number of finalizers, and kill all the weak pointers first...
n = 0;
for (w = list; w; w = w->link) {
StgArrWords *farr;
// Better not be a DEAD_WEAK at this stage; the garbage
// collector removes DEAD_WEAKs from the weak pointer list.
ASSERT(w->header.info != &stg_DEAD_WEAK_info);
if (w->finalizer != &stg_NO_FINALIZER_closure) {
n++;
}
farr = (StgArrWords *)UNTAG_CLOSURE(w->cfinalizer);
if ((StgClosure *)farr != &stg_NO_FINALIZER_closure)
runCFinalizer((void *)farr->payload[0],
(void *)farr->payload[1],
(void *)farr->payload[2],
farr->payload[3]);
#ifdef PROFILING
// A weak pointer is inherently used, so we do not need to call
// LDV_recordDead().
//
// Furthermore, when PROFILING is turned on, dead weak
// pointers are exactly as large as weak pointers, so there is
// no need to fill the slop, either. See stg_DEAD_WEAK_info
// in StgMiscClosures.hc.
#endif
SET_HDR(w, &stg_DEAD_WEAK_info, w->header.prof.ccs);
}
if (task != NULL) {
task->running_finalizers = rtsFalse;
}
// No finalizers to run?
if (n == 0) return;
debugTrace(DEBUG_weak, "weak: batching %d finalizers", n);
size = n + mutArrPtrsCardTableSize(n);
arr = (StgMutArrPtrs *)allocate(cap, sizeofW(StgMutArrPtrs) + size);
TICK_ALLOC_PRIM(sizeofW(StgMutArrPtrs), n, 0);
SET_HDR(arr, &stg_MUT_ARR_PTRS_FROZEN_info, CCS_SYSTEM);
arr->ptrs = n;
arr->size = size;
n = 0;
for (w = list; w; w = w->link) {
if (w->finalizer != &stg_NO_FINALIZER_closure) {
arr->payload[n] = w->finalizer;
n++;
}
}
// set all the cards to 1
for (i = n; i < size; i++) {
arr->payload[i] = (StgClosure *)(W_)(-1);
}
t = createIOThread(cap,
RtsFlags.GcFlags.initialStkSize,
rts_apply(cap,
rts_apply(cap,
(StgClosure *)runFinalizerBatch_closure,
rts_mkInt(cap,n)),
(StgClosure *)arr)
);
scheduleThread(cap,t);
}
