error_t barrier_wait(struct barrier_s *barrier)
{
register uint_t event;
register void *listner;
register uint_t ticket;
register uint_t index;
register uint_t wqdbsz;
register wqdb_t *wqdb;
register struct thread_s *this;
uint_t irq_state;
uint_t tm_now;
tm_now = cpu_time_stamp();
this = current_thread;
index = this->info.order;
if((barrier->signature != BARRIER_ID) || ((barrier->owner != NULL) && (barrier->owner != this->task)))
return EINVAL;
wqdbsz = PMM_PAGE_SIZE / sizeof(wqdb_record_t);
wqdb = barrier->wqdb_tbl[index / wqdbsz];
#if !(CONFIG_USE_SCHED_LOCKS)
event = sched_event_make (this, SCHED_OP_WAKEUP);
listner = sched_get_listner(this, SCHED_OP_WAKEUP);
#else
listner = (void*)this;
#endif
wqdb->tbl[index % wqdbsz].event = event;
wqdb->tbl[index % wqdbsz].listner = listner;
#if CONFIG_BARRIER_ACTIVE_WAIT
register uint_t current_phase;
current_phase = barrier->phase;
#endif /* CONFIG_BARRIER_ACTIVE_WAIT */
cpu_disable_all_irq(&irq_state);
ticket = arch_barrier_wait(barrier->cluster, barrier->hwid);
cpu_restore_irq(irq_state);
if(ticket < 0) return EINVAL;
if(ticket == barrier->count)
barrier->tm_first = tm_now;
else if(ticket == 1)
barrier->tm_last = tm_now;
#if CONFIG_BARRIER_ACTIVE_WAIT
while(cpu_uncached_read(&barrier->state[current_phase]) == 0)
sched_yield(this);
#else
sched_sleep(this);
#endif /* CONFIG_BARRIER_ACTIVE_WAIT */
return (ticket == 1) ? PTHREAD_BARRIER_SERIAL_THREAD : 0;
}
//inline
error_t remote_fifo_put(struct remote_fifo_s *remote_fifo, cid_t cid, void *item)
{
size_t wridx;
size_t rdidx;
size_t total_slot_nbr;
uint_t irq_state;
#if RF_PRINT
uint32_t start;
uint32_t end;
start = cpu_time_stamp(); //cpu_get_ticks(current_cpu);
#endif
total_slot_nbr = remote_lw((void*)&remote_fifo->slot_nbr, cid);
//assert(size);//if the message is bigger than cacheline, it could wrap arround the fifo => msg is not countigius
mcs_lock_remote(&remote_fifo->lock, cid, &irq_state);
wridx = remote_lw((void*)&remote_fifo->wridx, cid);
rdidx = remote_lw((void*)&remote_fifo->rdidx, cid);
if(((wridx + 1) % total_slot_nbr) == rdidx)
{
mcs_unlock_remote(&remote_fifo->lock, cid, irq_state);
return EAGAIN;
}
item_set(remote_fifo, cid, item, wridx);
remote_sw((void*)&remote_fifo->wridx, cid,
(wridx + 1) % total_slot_nbr);
mcs_unlock_remote(&remote_fifo->lock, cid, irq_state);
#if RF_PRINT
end = cpu_time_stamp();
printk(INFO, "[%d] %s: posting in cid %d at %d\n",
cpu_get_id(), __FUNCTION__, cid, end-start);
#endif
return 0;
}
int sys_fork(uint_t flags, uint_t cpu_gid)
{
fork_info_t info;
struct dqdt_attr_s attr;
struct thread_s *this_thread;
struct task_s *this_task;
struct thread_s *child_thread;
struct task_s *child_task;
uint_t irq_state;
uint_t cpu_lid;
uint_t cid;
error_t err;
uint_t tm_start;
uint_t tm_end;
uint_t tm_bRemote;
uint_t tm_aRemote;
tm_start = cpu_time_stamp();
fork_dmsg(1, "%s: cpu %d, started [%d]\n",
__FUNCTION__,
cpu_get_id(),
tm_start);
this_thread = current_thread;
this_task = this_thread->task;
info.current_clstr = current_cluster;
err = atomic_add(&this_task->childs_nr, 1);
if(err >= CONFIG_TASK_CHILDS_MAX_NR)
{
err = EAGAIN;
goto fail_childs_nr;
}
fork_dmsg(1, "%s: task of pid %d can fork a child [%d]\n",
__FUNCTION__,
this_task->pid,
cpu_time_stamp());
info.isDone = false;
info.this_thread = this_thread;
info.this_task = this_task;
info.flags = flags;
cpu_disable_all_irq(&irq_state);
cpu_restore_irq(irq_state);
if(current_cpu->fpu_owner == this_thread)
{
fork_dmsg(1, "%s: going to save FPU\n", __FUNCTION__);
cpu_fpu_context_save(&this_thread->uzone);
}
if(flags & PT_FORK_USE_TARGET_CPU)
{
cpu_gid = cpu_gid % arch_onln_cpu_nr();
cpu_lid = arch_cpu_lid(cpu_gid);
cid = arch_cpu_cid(cpu_gid);
attr.cid = cid;
attr.cpu_id = arch_cpu_lid(cpu_gid);
info.isPinned = true;
}
else
{
info.isPinned = false;
dqdt_attr_init(&attr, NULL);
err = dqdt_task_placement(dqdt_root, &attr);
}
info.cpu = cpu_lid2ptr(attr.cpu_id);
info.cid_exec = attr.cid_exec;
/* Keeps the first two processes on current cluster. This is used by cluster zero to keep
* the "sh" process on this cluster. Init is forced on current_cluster in the
* task_load_init() function.
*/
if ( this_task->pid < PID_MIN_GLOBAL+2 )
info.cid_exec = current_cid;
fork_dmsg(1, "%s: new task will be placed on cluster %d, cpu %d. Task will be moved on cluster %u on exec()\n", \
__FUNCTION__, attr.cid, attr.cpu_id, info.cid_exec);
tm_bRemote = cpu_time_stamp();
err = do_fork(&info);
tm_aRemote = cpu_time_stamp();
if(err)
goto fail_do_fork;
child_thread = info.child_thread;
child_task = info.child_task;
spinlock_lock(&this_task->lock);
list_add(&this_task->children, &child_task->list);
spinlock_unlock(&this_task->lock);
fork_dmsg(1, "%s: childs (task & thread) have been registered in their parents lists [%d]\n",
__FUNCTION__,
cpu_time_stamp());
fork_dmsg(1, "%s: going to add child to target scheduler\n", __FUNCTION__);
sched_add_created(child_thread);
tm_end = cpu_time_stamp();
fork_dmsg(1, "%s: cpu %d, pid %d, done [s:%u, bR:%u, aR:%u, e:%u, d:%u, t:%u, r:%u]\n",
__FUNCTION__,
cpu_get_id(),
this_task->pid,
tm_start,
tm_bRemote,
tm_aRemote,
tm_end,
attr.tm_request,
tm_end - tm_start,
info.tm_event);
return child_task->pid;
fail_do_fork:
fail_childs_nr:
atomic_add(&this_task->childs_nr, -1);
this_thread->info.errno = err;
return -1;
}
error_t do_fork(fork_info_t *info)
{
kmem_req_t req;
struct dqdt_attr_s attr;
struct thread_s *child_thread;
struct task_s *child_task;
struct page_s *page;
uint_t cid;
error_t err;
sint_t order;
fork_dmsg(1, "%s: cpu %d, started [%d]\n",
__FUNCTION__,
cpu_get_id(),
cpu_time_stamp());
child_thread = NULL;
child_task = NULL;
page = NULL;
cid = info->cpu->cluster->id;
attr.cid = cid;
attr.cpu_id = 0;
attr.cid_exec = info->cid_exec;
//dqdt_update_threads_number(attr.cluster->levels_tbl[0], attr.cpu->lid, 1);
dqdt_update_threads_number(cid, attr.cpu_id, 1);
//attr.cluster = info->current_clstr;
attr.cid = cid;
err = task_create(&child_task, &attr, CPU_USR_MODE);
//attr.cluster = info->cpu->cluster;
attr.cid = cid;
if(err) goto fail_task;
fork_dmsg(1, "%s: cpu %d, ppid %d, task @0x%x, pid %d, task @0x%x [%d]\n",
__FUNCTION__,
cpu_get_id(),
info->this_task->pid,
info->this_task,
child_task->pid,
child_task,
cpu_time_stamp());
req.type = KMEM_PAGE;
req.size = ARCH_THREAD_PAGE_ORDER;
req.flags = AF_KERNEL | AF_REMOTE;
req.ptr = info->cpu->cluster;
req.ptr = info->current_clstr;
page = kmem_alloc(&req);
if(page == NULL)
goto fail_mem;
fork_dmsg(1, "%s: child pid will be %d on cluster %d, cpu %d [%d]\n",
__FUNCTION__,
child_task->pid,
child_task->cpu->cluster->id,
child_task->cpu->gid,
cpu_time_stamp());
err = task_dup(child_task, info->this_task);
if(err) goto fail_task_dup;
signal_manager_destroy(child_task);
signal_manager_init(child_task);
fork_dmsg(1, "%s: parent task has been duplicated [%d]\n",
__FUNCTION__,
cpu_time_stamp());
child_task->current_clstr = info->current_clstr;
err = vmm_dup(&child_task->vmm, &info->this_task->vmm);
if(err) goto fail_vmm_dup;
fork_dmsg(1, "%s: parent vmm has been duplicated [%d]\n",
__FUNCTION__,
cpu_time_stamp());
child_thread = (struct thread_s*) ppm_page2addr(page);
/* Set the child page before calling thread_dup */
child_thread->info.page = page;
err = thread_dup(child_task,
child_thread,
info->cpu,
info->cpu->cluster,
info->this_thread);
if(err) goto fail_thread_dup;
/* Adjust child_thread attributes */
if(info->flags & PT_FORK_USE_AFFINITY)
{
child_thread->info.attr.flags |= (info->flags & ~(PT_ATTR_LEGACY_MASK));
if(!(info->flags & PT_ATTR_MEM_PRIO))
child_thread->info.attr.flags &= ~(PT_ATTR_MEM_PRIO);
if(!(info->flags & PT_ATTR_AUTO_MGRT))
child_thread->info.attr.flags &= ~(PT_ATTR_AUTO_MGRT);
if(!(info->flags & PT_ATTR_AUTO_NXTT))
child_thread->info.attr.flags &= ~(PT_ATTR_AUTO_NXTT);
}
fork_dmsg(1, "%s: parent current thread has been duplicated, tid %x [%d]\n",
__FUNCTION__,
child_thread,
cpu_time_stamp());
if(info->isPinned)
thread_migration_disabled(child_thread);
else
thread_migration_enabled(child_thread);
list_add_last(&child_task->th_root, &child_thread->rope);
child_task->threads_count = 1;
child_task->threads_nr ++;
child_task->state = TASK_READY;
order = bitmap_ffs2(child_task->bitmap, 0, sizeof(child_task->bitmap));
if(order == -1) goto fail_order;
bitmap_clear(child_task->bitmap, order);
child_thread->info.attr.key = order;
child_thread->info.order = order;
child_task->next_order = order + 1;
child_task->max_order = order;
child_task->uid = info->this_task->uid;
child_task->parent = info->this_task->pid;
err = sched_register(child_thread);
assert(err == 0);
cpu_context_set_tid(&child_thread->info.pss, (reg_t)child_thread);
cpu_context_set_pmm(&child_thread->info.pss, &child_task->vmm.pmm);
cpu_context_dup_finlize(&child_thread->pws, &child_thread->info.pss);
child_thread->info.retval = 0;
child_thread->info.errno = 0;
info->child_thread = child_thread;
info->child_task = child_task;
return 0;
fail_order:
fail_thread_dup:
fail_vmm_dup:
fail_task_dup:
printk(WARNING, "WARNING: %s: destroy child thread\n", __FUNCTION__);
req.ptr = page;
kmem_free(&req);
fail_mem:
fail_task:
//FIXME
//dqdt_update_threads_number(attr.cluster->levels_tbl[0], attr.cpu->lid, -1);
dqdt_update_threads_number(attr.cid, attr.cpu_id, -1);
printk(WARNING, "WARNING: %s: destroy child task\n", __FUNCTION__);
if(child_task != NULL)
task_destroy(child_task);
printk(WARNING, "WARNING: %s: fork err %d [%d]\n",
__FUNCTION__,
err,
cpu_time_stamp());
return err;
}
static void barrier_do_broadcast(struct barrier_s *barrier)
{
register uint_t tm_first;
register uint_t tm_last;
register uint_t tm_start;
register uint_t wqdbsz;
register uint_t tm_end;
register uint_t ticket;
register uint_t index;
register uint_t count;
register uint_t event;
register void *listner;
register wqdb_t *wqdb;
register uint_t i;
tm_start = cpu_time_stamp();
tm_first = barrier->tm_first;
tm_last = barrier->tm_last;
wqdbsz = PMM_PAGE_SIZE / sizeof(wqdb_record_t);
ticket = 0;
#if ARCH_HAS_BARRIERS
count = barrier->count;
#else
count = barrier->count - 1; /* last don't sleep */
#endif
for(index = 0; ((index < BARRIER_WQDB_NR) && (ticket < count)); index++)
{
wqdb = barrier->wqdb_tbl[index];
for(i = 0; ((i < wqdbsz) && (ticket < count)); i++)
{
#if CONFIG_BARRIER_BORADCAST_UREAD
event = cpu_uncached_read(&wqdb->tbl[i].event);
listner = (void*) cpu_uncached_read(&wqdb->tbl[i].listner);
#else
event = wqdb->tbl[i].event;
listner = wqdb->tbl[i].listner;
#endif
if(listner != NULL)
{
wqdb->tbl[i].listner = NULL;
#if CONFIG_USE_SCHED_LOCKS
sched_wakeup((struct thread_s*) listner);
#else
sched_event_send(listner, event);
#endif
ticket ++;
}
}
}
tm_end = cpu_time_stamp();
printk(INFO, "INFO: %s: cpu %d [F: %d, L: %d, B: %d, E: %d, T: %d]\n",
__FUNCTION__,
cpu_get_id(),
tm_first,
tm_last,
tm_start,
tm_end,
tm_end - tm_first);
}
/* TODO: reintroduce barrier's ops to deal with case-specific treatment */
error_t barrier_wait(struct barrier_s *barrier)
{
register uint_t ticket;
register uint_t index;
register uint_t wqdbsz;
register wqdb_t *wqdb;
register bool_t isShared;
struct thread_s *this;
uint_t tm_now;
tm_now = cpu_time_stamp();
this = current_thread;
index = this->info.order;
ticket = 0;
isShared = (barrier->owner == NULL) ? true : false;
if((barrier->signature != BARRIER_ID) || ((isShared == false) && (barrier->owner != this->task)))
return EINVAL;
wqdbsz = PMM_PAGE_SIZE / sizeof(wqdb_record_t);
if(isShared)
{
spinlock_lock(&barrier->lock);
index = barrier->index ++;
ticket = barrier->count - index;
}
wqdb = barrier->wqdb_tbl[index / wqdbsz];
#if CONFIG_USE_SCHED_LOCKS
wqdb->tbl[index % wqdbsz].listner = (void*)this;
#else
uint_t irq_state;
cpu_disable_all_irq(&irq_state); /* To prevent against any scheduler intervention */
wqdb->tbl[index % wqdbsz].event = sched_event_make (this, SCHED_OP_WAKEUP);
wqdb->tbl[index % wqdbsz].listner = sched_get_listner(this, SCHED_OP_WAKEUP);
#endif
if(isShared == false)
ticket = atomic_add(&barrier->waiting, -1);
if(ticket == 1)
{
#if !(CONFIG_USE_SCHED_LOCKS)
cpu_restore_irq(irq_state);
#endif
barrier->tm_last = tm_now;
wqdb->tbl[index % wqdbsz].listner = NULL;
if(isShared)
{
barrier->index = 0;
spinlock_unlock(&barrier->lock);
}
else
atomic_init(&barrier->waiting, barrier->count);
barrier_do_broadcast(barrier);
return PTHREAD_BARRIER_SERIAL_THREAD;
}
if(ticket == barrier->count)
barrier->tm_first = tm_now;
spinlock_unlock_nosched(&barrier->lock);
sched_sleep(this);
#if !(CONFIG_USE_SCHED_LOCKS)
cpu_restore_irq(irq_state);
#endif
return 0;
}
void* kvfsd(void *arg)
{
uint_t tm_now, cntr;
struct task_s *task;
struct thread_s *this;
struct cpu_s *cpu;
struct alarm_info_s info;
struct event_s event;
uint_t fs_type;
error_t err;
cpu_enable_all_irq(NULL);
printk(INFO, "INFO: Starting KVFSD on CPU %d [ %d ]\n", cpu_get_id(), cpu_time_stamp());
task = current_task;
fs_type = VFS_TYPES_NR;
#if CONFIG_ROOTFS_IS_EXT2
fs_type = VFS_EXT2_TYPE;
#endif
#if CONFIG_ROOTFS_IS_VFAT
#if CONFIG_ROOTFS_IS_EXT2
#error More than one root fs has been selected
#endif
fs_type = VFS_VFAT_TYPE;
#endif /* CONFIG_ROOTFS_IS_VFAT_TYPE */
err = vfs_init(__sys_blk,
fs_type,
VFS_MAX_NODE_NUMBER,
VFS_MAX_FILE_NUMBER,
&task->vfs_root);
task->vfs_cwd = task->vfs_root;
printk(INFO, "INFO: Virtual File System (VFS) Is Ready\n");
sysconf_init();
if(err == 0)
{
if((err = task_load_init(task)))
{
printk(WARNING, "WARNING: failed to load user process, err %d [%u]\n",
err,
cpu_time_stamp());
}
}
#if CONFIG_DEV_VERSION
if(err != 0)
{
struct thread_s *thread;
printk(INFO, "INFO: Creating kernel level terminal\n");
thread = kthread_create(task,
&kMiniShelld,
NULL,
current_cluster->id,
current_cpu->lid);
thread->task = task;
list_add_last(&task->th_root, &thread->rope);
err = sched_register(thread);
assert(err == 0);
sched_add_created(thread);
}
#endif
this = current_thread;
cpu = current_cpu;
event_set_senderId(&event, this);
event_set_priority(&event, E_FUNC);
event_set_handler(&event, &kvfsd_alarm_event_handler);
info.event = &event;
cntr = 0;
while(1)
{
alarm_wait(&info, 10);
sched_sleep(this);
tm_now = cpu_time_stamp();
printk(INFO, "INFO: System Current TimeStamp %u\n", tm_now);
sync_all_pages();
if((cntr % 4) == 0)
dqdt_print_summary(dqdt_root);
cntr ++;
}
return NULL;
}
/*
* FIXME: define spinlock_rdlock() so all locking on task->th_lock
* becoms rdlock but on join/detach/destroy
*/
int sys_thread_wakeup(pthread_t tid, pthread_t *tid_tbl, uint_t count)
{
struct task_s *task;
struct thread_s *this;
struct thread_s *target;
pthread_t tbl[100];
void *listner;
uint_t event;
sint_t i;
error_t err;
this = current_thread;
task = this->task;
i = -1;
if(tid_tbl != NULL)
{
if((((uint_t)tid_tbl + (count*sizeof(pthread_t))) >= CONFIG_KERNEL_OFFSET) ||
(count == 0) || (count > 100))
{
err = -1;
goto fail_tid_tbl;
}
if((err = cpu_uspace_copy(&tbl[0], tid_tbl, sizeof(pthread_t*) * count)))
goto fail_usapce;
if(tbl[0] != tid)
{
err = -2;
goto fail_first_tid;
}
}
else
{
count = 1;
tbl[0] = tid;
}
for(i = 0; i < count; i++)
{
tid = tbl[i];
if(tid > task->max_order)
{
err = -3;
goto fail_tid;
}
target = task->th_tbl[tid];
if((target == NULL) || (target->signature != THREAD_ID))
{
err = -4;
goto fail_target;
}
listner = sched_get_listner(target, SCHED_OP_UWAKEUP);
event = sched_event_make(target,SCHED_OP_UWAKEUP);
if(this->info.isTraced == true)
{
printk(INFO,"%s: tid %d --> tid %d [%d][%d]\n",
__FUNCTION__,
this->info.order,
tid,
cpu_time_stamp(),
i);
}
sched_event_send(listner,event);
cpu_wbflush();
}
return 0;
fail_target:
fail_tid:
fail_first_tid:
fail_usapce:
fail_tid_tbl:
printk(INFO, "%s: cpu %d, pid %d, tid %x, i %d, count %d, ttid %x, request has failed with err %d [%d]\n",
__FUNCTION__,
cpu_get_id(),
task->pid,
this,
i,
count,
tid,
err,
cpu_time_stamp());
this->info.errno = EINVAL;
return -1;
}