Пример #1
0
void* thread_dispatcher( void *a )
{
	Huint	  tid;
    argument  *arg;

	tid = hthread_self();
	arg = (argument*)a;

	printf( "Starting Dispatcher Thread:   (TID=%u)\n", tid );
	while( 1 )
    {
        hthread_mutex_lock( arg->mutex );

        while( arg->jobs >= 10 )
        {
            //printf( "Queue Full:   (JOBS=%u)\n", arg->jobs );
            hthread_cond_wait( arg->cond_notfull, arg->mutex );
        }

	    printf( "Creating Job:        (TID=%u) (JOBS=%u)\n", tid, arg->jobs );
        arg->jobs = arg->jobs + 1;

	    //printf( "Queue Not Empty: (TID=%u) (JOBS=%u)\n", tid, arg->jobs );
        hthread_cond_signal( arg->cond_notempty );

        hthread_yield();
        hthread_mutex_unlock( arg->mutex );
    }
    
	printf( "Exiting Dispatcher Thread:    (TID=%u)\n", tid );
	return NULL;
}
Пример #2
0
void* _bootstrap_thread( hthread_start_t func, void *arg )
{
    void *ret;
    
#ifdef HTHREADS_SMP
    while(!_release_syscall_lock());
#endif

    // Get start time
    hthread_time_t start = hthread_time_get();

    // Invoke  the start function and grab the return value
    ret = func( arg );
    
    // Get stop time and write execution time in TCB structure
    hthread_time_t stop = hthread_time_get();
    Huint tid = hthread_self();
    threads[tid].execution_time = stop-start;
     
    // Decrement the counter. It is safer to do this 
    // after hthread_exit but since we don't return
    // from this call, we will do it here.
    thread_counter--;
    
    // Exit the thread using the return value
    hthread_exit( ret );

    // This statement should never be reached
    return NULL;
}
Пример #3
0
void* simple_thread(void * arg) 
{
    struct simple_test * simple;
	
    // Extract thread argument
    simple = (struct simple_test *) arg;

#ifdef PRINT	
    int * pic_0_addr = (void*)0x41200000;
    int * pic_1_addr = (void*)0x41210000;
    printf( "\n..................................\n");
    printf( "simple_thread: CPUID = %d\n", _get_procid());
    printf( "simple_thread: PIC 0 = 0x%8.8x\n",*pic_0_addr);
    printf( "simple_thread: PIC 1 = 0x%8.8x\n",*pic_1_addr);
    printf( "simple_thread: Val =  %d\n", simple->val );
#endif
    
    // Increment value
    simple->val = simple->val + 10; 

#ifdef YIELD    
    hthread_yield();
#endif

#ifdef SELF
    hthread_t self_test = hthread_self();
#ifdef PRINT
    printf("simple_thread: TID VERIFIED %d\n", self_test);
#endif
#endif		

    return (void*)simple->val;
}
Пример #4
0
void * measureThread (void * arg)
{
    thread_arg_t * my_arg;
    my_arg = (thread_arg_t *)arg;
    Huint tid;
    volatile Hint * counter = &my_arg->counter;

    tid = hthread_self();

    // Pre-lock blocking mutex
    dbg_printf("MEASURE THREAD : Timing lock of block mutex\n");
    my_arg->lock_start = readTimer(  );
    hthread_mutex_lock(&my_arg->block_mutex);
    my_arg->lock_stop = readTimer(  );

    // Wait for all threads to block (i.e. wait for counter to reach limit)
    dbg_printf("### Waiting for counter value to be reached ###\n");
    while (*counter < my_arg->num_load_threads)
    {
        dbg_printf("Measurement thread yielding...(counter = %d)\n",my_arg->counter);
        //hthread_yield();
    }
    dbg_printf("### Counter value reached ###\n");

    // (**B**) Unlock blocking mutex
    dbg_printf("MEASURE THREAD : Timing unlock of block mutex\n");
    my_arg->unlock_start = readTimer(  );
    hthread_mutex_unlock(&my_arg->block_mutex);
    my_arg->unlock_stop = readTimer(  );

    //hthread_exit( NULL );
	return NULL;
}
Пример #5
0
int main( int argc, char *argv[] )
{
    Huint       j;
    Huint       self;
    register Huint start_time;
    register Huint end_time;

    log_create( &logbuf, LOOPS * 2 );
    for( j = 0; j < LOOPS; j++ )
    {
        start_time = timer_get_globallo();
        self = hthread_self();
        end_time = timer_get_globallo();
        
        logbuf.buffer[ logbuf.pos++ ] = start_time;
        logbuf.buffer[ logbuf.pos++ ] = end_time;
        
#ifdef PRINT
            //XExc_mDisableExceptions(XEXC_NON_CRITICAL);
            printf( "Self: %u\n", self );
            //XExc_mEnableExceptions(XEXC_NON_CRITICAL);
#endif
    }

    //XExc_mDisableExceptions(XEXC_NON_CRITICAL);
    printf( "Flushing %u timing values...\n", logbuf.pos );
    log_flush( &logbuf );
    //log_close( &logbuf );
	printf( "--DONE--\n" );
    //XExc_mEnableExceptions(XEXC_NON_CRITICAL);

	return 1;
}
Пример #6
0
/*-------------------------------------------------------------------*/
DLL_EXPORT int  hthread_obtain_wrlock( RWLOCK* plk, const char* location )
{
    int rc;
    U64 waitdur;
    ILOCK* ilk;
    TIMEVAL tv;
    ilk = (ILOCK*) plk->ilk;
    PTTRACE( "wrlock before", plk, NULL, location, PTT_MAGIC );
    rc = hthread_rwlock_trywrlock( &ilk->rwlock );
    if (EBUSY == rc)
    {
        waitdur = host_tod();
        rc = hthread_rwlock_wrlock( &ilk->rwlock );
        gettimeofday( &tv, NULL );
        waitdur = host_tod() - waitdur;
    }
    else
    {
        gettimeofday( &tv, NULL );
        waitdur = 0;
    }
    PTTRACE2( "wrlock after", plk, (void*) waitdur, location, rc, &tv );
    if (rc)
        loglock( ilk, rc, "obtain_wrlock", location );
    if (!rc || EOWNERDEAD == rc)
    {
        hthread_mutex_lock( &ilk->locklock );
        ilk->location = location;
        ilk->tid = hthread_self();
        memcpy( &ilk->time, &tv, sizeof( TIMEVAL ));
        hthread_mutex_unlock( &ilk->locklock );
    }
    return rc;
}
Пример #7
0
/*-------------------------------------------------------------------*/
DLL_EXPORT TID  hthread_thread_id( const char* location )
{
    TID tid;
    UNREFERENCED( location );
    tid = hthread_self();
    return tid;
}
Пример #8
0
/*-------------------------------------------------------------------*/
DLL_EXPORT void hthread_exit_thread( void* rc, const char* location )
{
    TID tid;
    tid = hthread_self();
    hthread_list_abandoned_locks( tid, location );
    hthread_exit( rc );
}
Пример #9
0
void* simpleThread( void *arg )
{
    uint thread_ctr;
    uint i;
    Huint thread_id;
    hthread_time_t t;

    thread_ctr = (uint)arg;
	thread_id = hthread_self();

    for (i=0; i<THREAD_ITERATIONS; i++)
    {
        printf("In Thread:  (TID= %u) (CTR= %u)\n", thread_id, thread_ctr);

        // idle for one second
        t = hthread_time_get() + CLOCKS_PER_SEC;
        while(hthread_time_get() < t);

        hthread_yield();
    }

    printf( "Exiting Thread:  (TID= %u) (CTR= %u)\n", thread_id, thread_ctr);
    hthread_exit(NULL);

    printf( "??? CODE AFTER THREAD EXIT, should not get here.\n");
    return NULL;
}
Пример #10
0
/*-------------------------------------------------------------------*/
static void loglock( ILOCK* ilk, const int rc, const char* calltype,
                                               const char* err_loc )
{
    const char* err_desc;

    switch (rc)
    {
        case EAGAIN:          err_desc = "max recursion";    break;
        case EPERM:           err_desc = "not owned";        break;
        case EINVAL:          err_desc = "invalid argument"; break;
        case EDEADLK:         err_desc = "deadlock";         break;
        case ENOTRECOVERABLE: err_desc = "not recoverable";  break;
        case EOWNERDEAD:      err_desc = "owner dead";       break;
        case EBUSY:           err_desc = "busy";             break; /* (should not occur) */
        case ETIMEDOUT:       err_desc = "timeout";          break; /* (should not occur) */
        default:              err_desc = "(unknown)";        break;
    }

    // "'%s(%s)' failed: rc=%d: %s; tid="TIDPAT", loc=%s"
    WRMSG( HHC90013, "E", calltype, ilk->name, rc, err_desc,
        hthread_self(), TRIMLOC( err_loc ));

    if (ilk->tid)
    {
        // "lock %s was obtained by thread "TIDPAT" at %s"
        WRMSG( HHC90014, "I", ilk->name, ilk->tid, TRIMLOC( ilk->location ));
    }
}
Пример #11
0
void * testThreadWithMeasurement (void * arg)
{
    thread_arg_t * my_arg;
    my_arg = (thread_arg_t *)arg;
    Huint tid;

    tid = hthread_self();

    // Atomically increment counter (protected by data_mutex)
    dbg_printf("LOAD w/ measure TID %d, incrementing counter (%d -> %d) \n",tid,my_arg->counter,my_arg->counter+1);
    hthread_mutex_lock(&my_arg->data_mutex);
    my_arg->counter = my_arg->counter + 1;
    hthread_mutex_unlock(&my_arg->data_mutex);

    // Grab and release block mutex (protected by block_mutex) - should be pre-locked by main thread
    dbg_printf("LOAD w/ measure TID %d, locking block mutex\n",tid);
    my_arg->measure_lock_start = readTimer(  );
    hthread_mutex_lock(&my_arg->block_mutex);
    my_arg->measure_lock_stop = readTimer(  );
    dbg_printf("LOAD w/ measure TID %d, unlocking block mutex\n",tid);
    hthread_mutex_unlock(&my_arg->block_mutex);

    //hthread_exit( NULL );
	return NULL;
}
Пример #12
0
void* thread_worker( void *a )
{
	Huint	  tid;
    argument  *arg;

	tid = hthread_self();
	arg = (argument*)a;

	printf( "Starting Worker Thread:   (TID=%u)\n", tid );
    while( 1 )
	{
        hthread_mutex_lock( arg->mutex );
        
        while( arg->jobs == 0 )
        {
            //printf( "Queue Empty: (TID=%u)\n", tid );
            hthread_cond_wait( arg->cond_notempty, arg->mutex );
        }
        
        arg->jobs = arg->jobs - 1;

		//printf( "Queue Not Full: (TID=%u) (JOBS=%u)\n",tid, arg->jobs );
        hthread_cond_signal( arg->cond_notfull );
        
		printf( "Running Job:        (TID=%u) (JOBS=%u)\n",tid,arg->jobs );
        hthread_yield();
        hthread_mutex_unlock( arg->mutex );
	}
	
	printf( "Exiting Worker Thread:    (TID=%u)\n", tid );
	return NULL;
}
Пример #13
0
void* child( void *arg )
{
    Huint           i;
    hthread_t       self;
	hthread_mutex_t *mutex;
    
    mutex = (hthread_mutex_t*)arg;
    self = hthread_self();
    
    for( i = 0; i < LOOP; i++ )
    {
        hthread_mutex_lock( mutex );
        
#ifdef PRINT
        printf( "Locked Mutex: %u %u\n", self, i );
#endif
//        hthread_yield();
        
        hthread_mutex_unlock( mutex );
        printf( "Yielding: %u ...\n", self );
        hthread_yield();
    }
	
	return NULL;
}
Пример #14
0
// Return the priority of the currently-running thread.
Huint get_priority(void)
{
    struct sched_param pr;
    Hint pol;
    hthread_getschedparam(hthread_self(), &pol, &pr);
    Huint priority = pr.sched_priority;
    return priority;
}
Пример #15
0
/*-------------------------------------------------------------------*/
static void* hthread_func( void* arg2 )
{
    THREAD_FUNC*  pfn  = (THREAD_FUNC*) *((void**)arg2+0);
    void*         arg  = (void*)        *((void**)arg2+1);
    TID           tid  = hthread_self();
    void*         rc;
    free( arg2 );
    rc = pfn( arg );
    hthread_list_abandoned_locks( tid, NULL );
    return rc;
}
void * foo_thread (void * arg) {
    // Extract TID
	unsigned int tid = hthread_self();

    // Use it to get sceduling priority
    unsigned int priority = 0;
    hthread_getschedparam( (hthread_t) tid, NULL, (struct sched_param *) &priority);

    // Return scheduling priority (used only for verifying hardware threads)
	return (void *) priority;
}
Пример #17
0
void * a_thread_function(void * arg) {
	struct testdata * data = (struct testdata *) arg;
	
	printf( "Thread %d Running\n", (int)hthread_self() );
	
	hthread_mutex_lock( data->mutex );
	*(data->start_num) += 1;
	hthread_cond_wait( data->cond, data->mutex );
	*(data->waken_num) += 1;
	hthread_mutex_unlock( data->mutex);
	return NULL;
}
Пример #18
0
void * testThread ( void * arg ) {
	int retVal;
	hthread_mutex_t * mutex = (hthread_mutex_t *) arg;
	
	//Try to lock a locked mutex
	hthread_mutex_trylock( mutex );

	//Test that another thread owns the mutex	
	if ( _mutex_owner( mutex->num ) != hthread_self() )
		retVal = SUCCESS;
	else 
		retVal = FAILURE;
	
	hthread_exit( (void *) retVal );
	return NULL;
}
Пример #19
0
void * testThread ( void * arg ) {
	int retVal;
	hthread_mutex_t * mutex = (hthread_mutex_t *) arg;
	
	//Test that after trylock returns, on an unlocked mutex, the calling thread owns the mutex
	hthread_mutex_trylock( mutex );
	
	if ( _mutex_owner( mutex->num ) == hthread_self() )
		retVal = SUCCESS;
	else 
		retVal = FAILURE;
	
	hthread_mutex_unlock( mutex );
	
	hthread_exit( (void *) retVal );
	return NULL;
}
Пример #20
0
void* consumer( void *arg )
{
    Hint        num;
    Hint        tot;
    Hint        val;
    hthread_t   id; 

    // Cast the argument to a buffer structure
    buffer_t *data;
    data = (buffer_t*)arg;

    // Get the thread id of this thread
    id = hthread_self();

    // Print out that we are starting
    TRACE3_PRINTF( "CONSUMER %d: (OP=START)\n", (int)id );

    num = 0;
    while( 1 )
    {
        // Read a value out of the buffer
        tot = buffer_get( data, &val );

        // Check if there was an error reading the value
        if( tot < -1 )   DEBUG_PRINTF( "ERROR: (OP=BUFFER GET) (STA=0x%8.8x)\n", tot );

        // Check to see if we should exit
        if( tot == -1 ) break;

        // Print out a message
        TRACE4_PRINTF( "CONSUMER %d: (READ=%d) (TOT=%d) (NUM=%d)\n", (int)id, val, tot, num );

        // Increment the number of values that we have read
        num += 1;
    }

    // Print out that we are exiting
    TRACE3_PRINTF( "CONSUMER %d: (OP=EXIT)\n", (int)id );

    // Print out how many value that we consumed
    TRACE1_PRINTF( "CONSUMER %d: (OP=NUM) (VAL=%d)\n", (int)id, num );

    // Return the number of items we produced
    return (void*)num;
}
Пример #21
0
void *handle_requests_loop(void *data)
{
  int rs;
  struct task *task;
  struct threadpool * tp = (struct threadpool *)data;

  // Pre-lock mutex
  rs = hthread_mutex_lock(tp->task_queue_mutex);

  while (1) {
    // Check to see if there are any tasks to execute
    if (tp->total_tasks > 0) {
      // If so, then grab one
      task = get_task(tp);
      aprintf("TID %d, got task!\n",hthread_self());


      if (task) {
    	// If the task is valid, then release lock
	    rs = hthread_mutex_unlock(tp->task_queue_mutex);

    	// Execute task
	    execute_task(task);
    	free(task);

        // Yield to allow another thread to do some work if possible
        hthread_yield();

    	// Re-acquire for next round
	    rs = hthread_mutex_lock(tp->task_queue_mutex);
      } else {
    	// Otherwise, wait for tasks
	    rs = hthread_cond_wait(tp->active_task, tp->task_queue_mutex);
      }
    } else {
      // Release lock and processor, let someone else do some work
      hthread_mutex_unlock(tp->task_queue_mutex);
      hthread_yield();

      // Re-acquire
      hthread_mutex_lock(tp->task_queue_mutex);
    }
  }
  return (void*)99;
}
Пример #22
0
void* producer( void *arg )
{
    Hint        num;
    Hint        tot;
    hthread_t   id; 

    // Cast the argument to a buffer structure
    buffer_t *data;
    data = (buffer_t*)arg;

    // Get the thread id of this thread
    id = hthread_self();

    // Print out that we are starting
    TRACE3_PRINTF( "PRODUCER: %d: (OP=START)\n", (int)id );

    num = 0;
    while( 1 )
    {
        // Add a value into the buffer
        tot = buffer_put( data, num );

        // Check if there was an error adding the value
        if( tot < -1 )   DEBUG_PRINTF( "ERROR: (OP=BUFFER PUT) (STA=0x%8.8x)\n", tot );

        // Check to see if we should exit
        if( tot == -1 ) break;

        // Print out a message
        TRACE4_PRINTF( "PRODUCER %d: (SENT=%d) (TOT=%d) (NUM=%d)\n", (int)id, num, tot, num );

        // Increment the number of values that we have added
        num += 1;

    }

    // Print out that we are exiting
    TRACE3_PRINTF( "PRODUCER: %d: (OP=EXIT)\n", (int)id );

    // Print out how many values that we consumed
    TRACE1_PRINTF( "PRODUCER: %d: (OP=NUM) (VAL=%d)\n", (int)id, num );

    return (void*)num;
}
Пример #23
0
/*-------------------------------------------------------------------*/
DLL_EXPORT int  hthread_try_obtain_wrlock( RWLOCK* plk, const char* location )
{
    int rc;
    ILOCK* ilk;
    TIMEVAL tv;
    ilk = (ILOCK*) plk->ilk;
    PTTRACE( "trywr before", plk, NULL, location, PTT_MAGIC );
    rc = hthread_rwlock_trywrlock( &ilk->rwlock );
    gettimeofday( &tv, NULL );
    PTTRACE2( "trywr after", plk, NULL, location, rc, &tv );
    if (rc && EBUSY != rc)
        loglock( ilk, rc, "try_obtain_wrlock", location );
    if (!rc)
    {
        hthread_mutex_lock( &ilk->locklock );
        ilk->location = location;
        ilk->tid = hthread_self();
        memcpy( &ilk->time, &tv, sizeof( TIMEVAL ));
        hthread_mutex_unlock( &ilk->locklock );
    }
    return rc;
}
Пример #24
0
void* child( void *arg )
{
    hthread_t       self;
	hthread_mutex_t *mutex;
    
    mutex = (hthread_mutex_t*)arg;
    self = hthread_self();
    
    printf( "Starting Child: %u\n", self );
    while( 1 )
    {
        hthread_mutex_lock( mutex );
        
        hthread_yield();
        printf( "Locked Mutex: %u\n", self );
        hthread_yield();
        
        hthread_mutex_unlock( mutex );
        hthread_yield();
    }
	
	return NULL;
}
Пример #25
0
void *daemon_thread(void *arg)
{
    TID_t next_tid = 0;
    TID_t hw_tid   = 0;
    int rv, hw_ret_val, regCount;
    hthread_t junk_tid;
    flag hw_available, hw_done;

    sched_param_t my_priority;
    Hint my_policy;

    // Display the daemon's priority
    hthread_getschedparam(hthread_self(),&my_policy,&my_priority);
    //printf("Daemon priority = %d\n", my_priority.sched_priority);

    // Create a pointer to the Daemon's communication struct with main.
    DaemonComm *dc = (DaemonComm *)arg;

    // Create a buffer for software interpreters to export/import their state to.
    ExportBuffer export_buffer;
    ImportBuffer import_buffer;

    // The software_interpreter_list holds active threads that are being interpreted by
    // the software interpreter.
    TCBNode software_interpreter_list[MAX_SW_THREAD];

    // The cur_tcb and the tcb_index are used to reference TCBs in the software interpreter list.
    TCBNode *cur_tcb;
    TCB *new_tcb;
    int tcb_index;

    // Initialize TCB List
    for (cur_tcb = software_interpreter_list; cur_tcb < software_interpreter_list + MAX_SW_THREAD; cur_tcb++)
    {
        UNSET(cur_tcb->valid);
    }

    // Initialize hardware flags
    SET(hw_available);
    SET(hw_done);


    printf("    ...DAEMON running.\n");
    // The daemon will run forever. Currently we have no clean shutdown mechanism.
    while(1)
    {

        // Process each TCB in the software interpreter list.
        for (cur_tcb = software_interpreter_list; cur_tcb < software_interpreter_list + MAX_SW_THREAD; cur_tcb++)
        {
            // If the TCB is invalid, then skip it.
            if (! ISSET(cur_tcb->valid))
                continue;

            // If a thread is done interpreting, then print its return value and invalidate the TCB.
            if (ISSET(cur_tcb->entry.communication.control.done_interpreting))
            {
                printf("DAEMON: Thread id %u (running in SW) returned %d\n", cur_tcb->entry.tid, cur_tcb->entry.communication.data.return_value);
                UNSET(cur_tcb->valid);
                continue;
            }

            // If the thread is done exporting to hardware, then invalidate the TCB and start
            // the hardware interpretation process.
            if (ISSET(cur_tcb->entry.communication.control.done_exporting))
            {
                // TODO: If a software thread has finished exporting, but in the meantime a
                // a new "run this thread only in hardware" request has come in, we need to copy
                // the thread state from the export buffer *back* into a software thread to make
                // room for the hw->sw migration we're about to do.
                // If there is new bytecode available to execute, then process it.
                if (ISSET(dc->new_code) & ISSET(dc->in_hw))
                {
                    // Kick thread back into SW

                    // Find an invalid TCB.
                    for (tcb_index = 0; tcb_index < MAX_SW_THREAD; tcb_index++)
                    {
                        if (! ISSET(software_interpreter_list[tcb_index].valid))
                            break;
                    }

                    // If an available TCB is found, then tcb_index must be less
                    // then MAX_SW_THREAD.
                    if (tcb_index < MAX_SW_THREAD)
                    {
                        // If an invalid TCB exists, initialize it and start
                        // the software interpreter.

                        // Initialize software_interpreter_list[tcb_index]
                        new_tcb = &software_interpreter_list[tcb_index].entry;

                        new_tcb->tid = cur_tcb->entry.tid;
                        new_tcb->virtualization.base_addr = software_interpreter_list[tcb_index].memory;
                        new_tcb->communication.data.export_buffer_addr = & export_buffer;
                        new_tcb->communication.data.import_buffer_addr = & import_buffer;
                        UNSET(new_tcb->communication.control.done_interpreting);
                        UNSET(new_tcb->communication.control.start_exporting);
                        UNSET(new_tcb->communication.control.done_exporting);

                        SET(software_interpreter_list[tcb_index].valid);

                        // Copy program/state to interpreter memory space
                        for (regCount = 0; regCount < NUMBER_REGISTERS ; regCount++)
                        {
                            import_buffer.register_file[regCount] = export_buffer.register_file[regCount];
                        }
                        memcpy(software_interpreter_list[tcb_index].memory, cur_tcb->memory, export_buffer.register_file[SP]);

                        //start_software_interpreter();
                        rv = hthread_create(&junk_tid, NULL, interpreter_entry_point_import, (void *)&(software_interpreter_list[tcb_index].entry));
                    }
                    else
                    {
                        // If the software_interpreter_list is full, issue an error message.
                        fprintf(stderr, "Preallocated TCB list is full.\n");
                    }


                }
                else
                {
                    // Move the thread into HW

                    // Grab the SW thread's TID
                    hw_tid = cur_tcb->entry.tid;

                    // Migrate state from export buffer into HW interpreter
                    reset_HVM();
                    printf("DAEMON: Migrating thread id %u from SW to HW...",hw_tid);
                    import_state_HVM(&export_buffer, cur_tcb->memory);
                    printf("COMPLETE\n");

                    // Start HW interpreter execution
                    light_LED(hw_tid);
                    UNSET(hw_done);
                    run_HVM();

                    // Invalidate TCB
                    UNSET(cur_tcb->valid);
                }
            }
        }


        // Is the hardware done interpreting
        if (is_HVM_done())
        {
            // Export HVM state and grab return value
            light_LED(0);
            SET(hw_done);
            export_state_HVM();
            wait_export_complete_HVM();
            hw_ret_val = get_HVM_return_value();

            // Display return value
            printf("DAEMON: Thread id %u (running in HW) returned %d\n", hw_tid, hw_ret_val);

            // Check to see if any SW threads exist that can now be run in HW
            for (tcb_index = 0; tcb_index < MAX_SW_THREAD; tcb_index++)
            {
                if (software_interpreter_list[tcb_index].valid)
                    break;
            }

            // If a valid SW thread exists, begin it's export process so that it can be migrated (otherwise, make the HW available again)
            if (tcb_index < MAX_SW_THREAD) {
                SET(software_interpreter_list[tcb_index].entry.communication.control.start_exporting);
            }
            else {
                printf("DAEMON: HW is available for the taking!\n");
                SET(hw_available);
            }
        }


        // If there is new bytecode available to execute, then process it.
        if (ISSET(dc->new_code))
        {
            // Increment the TID counter
            next_tid++;

            // If told to run this thread in HW, check to see if we need to force the HW to be available (migrating a thread from HW to SW)
            if (ISSET(dc->in_hw))
            {
                // Check to see if HW is even available
                if (!ISSET(hw_available) & !ISSET(hw_done))
                {
                    // It's not, so we must make it available
                    // export hardware to software, if needed
                    printf("DAEMON: Migrating thread id %u from HW to SW...", hw_tid);

                    // Stop HVM and export its state
                    light_LED(0);
                    export_state_HVM();

                    //wait_export_complete_HVM(); // This function waits for exported PC to be all F's and this won't be the case in a pre-empted program
                    delay(99999999);  // Use a delay instead to wait for export process to finish

                    printf("COMPLETE\n");

                    // migrate HW state to import buffer (registers now, and program/stack later - just below)
                    migrate_HVM_registers_to_buffer(&import_buffer);

                    // create new SW based on import buffer

                    // Find an invalid TCB.
                    for (tcb_index = 0; tcb_index < MAX_SW_THREAD; tcb_index++)
                    {
                        if (! ISSET(software_interpreter_list[tcb_index].valid))
                            break;
                    }

                    // If an available TCB is found, then tcb_index must be less
                    // then MAX_SW_THREAD.
                    if (tcb_index < MAX_SW_THREAD)
                    {
                        // If an invalid TCB exists, initialize it and start
                        // the software interpreter.

                        // Initialize software_interpreter_list[tcb_index]
                        new_tcb = &software_interpreter_list[tcb_index].entry;

                        new_tcb->tid = hw_tid;
                        new_tcb->virtualization.base_addr = software_interpreter_list[tcb_index].memory;
                        new_tcb->communication.data.export_buffer_addr = & export_buffer;
                        new_tcb->communication.data.import_buffer_addr = & import_buffer;
                        UNSET(new_tcb->communication.control.done_interpreting);
                        UNSET(new_tcb->communication.control.start_exporting);
                        UNSET(new_tcb->communication.control.done_exporting);

                        SET(software_interpreter_list[tcb_index].valid);

                        // Copy program/state to interpreter memory space
                        //memcpy(software_interpreter_list[tcb_index].memory, dc->new_code_address, dc->new_code_size);
                        memcpy(software_interpreter_list[tcb_index].memory, hvm_prog_mem, get_current_SP_HVM());

                        //start_software_interpreter();
                        rv = hthread_create(&junk_tid, NULL, interpreter_entry_point_import, (void *)&(software_interpreter_list[tcb_index].entry));
                    }
                    else
                    {
                        // If the software_interpreter_list is full, issue an error message.
                        fprintf(stderr, "Preallocated TCB list is full.\n");
                    }

                    // Set HW available flag, so the new thread falls through and is created by the code below
                    SET(hw_available);
                }
                else if (!ISSET(hw_available) & ISSET(hw_done))
                {
                    // If its not available, but the thread in HW is complete, then there is no need to migrate the thread...
                    // Just change the availalbility flag for the code below to take care of
                    SET(hw_available);
                }
            }


            // TODO: I don't think we need this else, now.  The dc->in_hw flag just forces a
            // hw -> sw migration, making the hw available to the new thread.
            // If HW available, run the thread in HW
            if ISSET(hw_available)
            {
                // De-asser ready flag
                UNSET(hw_available);

                // Download fresh code
                hw_tid = next_tid;
                printf("DAEMON: Started thread id %u in HW\n", hw_tid);
                load_memory(hvm_prog_mem, dc->new_code_size, dc->new_code_address);

                // Reset and run the interpreter
                reset_HVM();
                light_LED(hw_tid);
                UNSET(hw_done);
                run_HVM();

            }
            // Otherwise run the thread in SW
            else
            {
                // This bytecode will run in software. Therefore it needs
                // to given a TCB in the software interpreter list.

                // Find an invalid TCB.
                for (tcb_index = 0; tcb_index < MAX_SW_THREAD; tcb_index++)
                {
                    if (! ISSET(software_interpreter_list[tcb_index].valid))
                        break;
                }

                // If an available TCB is found, then tcb_index must be less
                // then MAX_SW_THREAD.
                if (tcb_index < MAX_SW_THREAD)
                {
                    // If an invalid TCB exists, initialize it and start
                    // the software interpreter.

                    // Initialize software_interpreter_list[tcb_index]
                    new_tcb = &software_interpreter_list[tcb_index].entry;

                    new_tcb->tid = next_tid;
                    new_tcb->virtualization.base_addr = software_interpreter_list[tcb_index].memory;
                    new_tcb->communication.data.export_buffer_addr = & export_buffer;
                    new_tcb->communication.data.import_buffer_addr = & import_buffer;
                    UNSET(new_tcb->communication.control.done_interpreting);
                    UNSET(new_tcb->communication.control.start_exporting);
                    UNSET(new_tcb->communication.control.done_exporting);

                    SET(software_interpreter_list[tcb_index].valid);
                    memcpy(software_interpreter_list[tcb_index].memory, dc->new_code_address, dc->new_code_size);

                    //start_software_interpreter();
                    printf("DAEMON: Started thread id %u in SW\n", next_tid);
                    rv = hthread_create(&junk_tid, NULL, interpreter_entry_point, (void *)&(software_interpreter_list[tcb_index].entry));
                }
                else
                {
                    // If the software_interpreter_list is full, issue an error message.
                    fprintf(stderr, "Preallocated TCB list is full.\n");
                }
            }

            // Unset the new_code flag
            UNSET(dc->new_code);
        }