error_t barrier_wait(struct barrier_s *barrier) { register uint_t event; register void *listner; register uint_t ticket; register uint_t index; register uint_t wqdbsz; register wqdb_t *wqdb; register struct thread_s *this; uint_t irq_state; uint_t tm_now; tm_now = cpu_time_stamp(); this = current_thread; index = this->info.order; if((barrier->signature != BARRIER_ID) || ((barrier->owner != NULL) && (barrier->owner != this->task))) return EINVAL; wqdbsz = PMM_PAGE_SIZE / sizeof(wqdb_record_t); wqdb = barrier->wqdb_tbl[index / wqdbsz]; #if !(CONFIG_USE_SCHED_LOCKS) event = sched_event_make (this, SCHED_OP_WAKEUP); listner = sched_get_listner(this, SCHED_OP_WAKEUP); #else listner = (void*)this; #endif wqdb->tbl[index % wqdbsz].event = event; wqdb->tbl[index % wqdbsz].listner = listner; #if CONFIG_BARRIER_ACTIVE_WAIT register uint_t current_phase; current_phase = barrier->phase; #endif /* CONFIG_BARRIER_ACTIVE_WAIT */ cpu_disable_all_irq(&irq_state); ticket = arch_barrier_wait(barrier->cluster, barrier->hwid); cpu_restore_irq(irq_state); if(ticket < 0) return EINVAL; if(ticket == barrier->count) barrier->tm_first = tm_now; else if(ticket == 1) barrier->tm_last = tm_now; #if CONFIG_BARRIER_ACTIVE_WAIT while(cpu_uncached_read(&barrier->state[current_phase]) == 0) sched_yield(this); #else sched_sleep(this); #endif /* CONFIG_BARRIER_ACTIVE_WAIT */ return (ticket == 1) ? PTHREAD_BARRIER_SERIAL_THREAD : 0; }
//inline error_t remote_fifo_put(struct remote_fifo_s *remote_fifo, cid_t cid, void *item) { size_t wridx; size_t rdidx; size_t total_slot_nbr; uint_t irq_state; #if RF_PRINT uint32_t start; uint32_t end; start = cpu_time_stamp(); //cpu_get_ticks(current_cpu); #endif total_slot_nbr = remote_lw((void*)&remote_fifo->slot_nbr, cid); //assert(size);//if the message is bigger than cacheline, it could wrap arround the fifo => msg is not countigius mcs_lock_remote(&remote_fifo->lock, cid, &irq_state); wridx = remote_lw((void*)&remote_fifo->wridx, cid); rdidx = remote_lw((void*)&remote_fifo->rdidx, cid); if(((wridx + 1) % total_slot_nbr) == rdidx) { mcs_unlock_remote(&remote_fifo->lock, cid, irq_state); return EAGAIN; } item_set(remote_fifo, cid, item, wridx); remote_sw((void*)&remote_fifo->wridx, cid, (wridx + 1) % total_slot_nbr); mcs_unlock_remote(&remote_fifo->lock, cid, irq_state); #if RF_PRINT end = cpu_time_stamp(); printk(INFO, "[%d] %s: posting in cid %d at %d\n", cpu_get_id(), __FUNCTION__, cid, end-start); #endif return 0; }
int sys_fork(uint_t flags, uint_t cpu_gid) { fork_info_t info; struct dqdt_attr_s attr; struct thread_s *this_thread; struct task_s *this_task; struct thread_s *child_thread; struct task_s *child_task; uint_t irq_state; uint_t cpu_lid; uint_t cid; error_t err; uint_t tm_start; uint_t tm_end; uint_t tm_bRemote; uint_t tm_aRemote; tm_start = cpu_time_stamp(); fork_dmsg(1, "%s: cpu %d, started [%d]\n", __FUNCTION__, cpu_get_id(), tm_start); this_thread = current_thread; this_task = this_thread->task; info.current_clstr = current_cluster; err = atomic_add(&this_task->childs_nr, 1); if(err >= CONFIG_TASK_CHILDS_MAX_NR) { err = EAGAIN; goto fail_childs_nr; } fork_dmsg(1, "%s: task of pid %d can fork a child [%d]\n", __FUNCTION__, this_task->pid, cpu_time_stamp()); info.isDone = false; info.this_thread = this_thread; info.this_task = this_task; info.flags = flags; cpu_disable_all_irq(&irq_state); cpu_restore_irq(irq_state); if(current_cpu->fpu_owner == this_thread) { fork_dmsg(1, "%s: going to save FPU\n", __FUNCTION__); cpu_fpu_context_save(&this_thread->uzone); } if(flags & PT_FORK_USE_TARGET_CPU) { cpu_gid = cpu_gid % arch_onln_cpu_nr(); cpu_lid = arch_cpu_lid(cpu_gid); cid = arch_cpu_cid(cpu_gid); attr.cid = cid; attr.cpu_id = arch_cpu_lid(cpu_gid); info.isPinned = true; } else { info.isPinned = false; dqdt_attr_init(&attr, NULL); err = dqdt_task_placement(dqdt_root, &attr); } info.cpu = cpu_lid2ptr(attr.cpu_id); info.cid_exec = attr.cid_exec; /* Keeps the first two processes on current cluster. This is used by cluster zero to keep * the "sh" process on this cluster. Init is forced on current_cluster in the * task_load_init() function. */ if ( this_task->pid < PID_MIN_GLOBAL+2 ) info.cid_exec = current_cid; fork_dmsg(1, "%s: new task will be placed on cluster %d, cpu %d. Task will be moved on cluster %u on exec()\n", \ __FUNCTION__, attr.cid, attr.cpu_id, info.cid_exec); tm_bRemote = cpu_time_stamp(); err = do_fork(&info); tm_aRemote = cpu_time_stamp(); if(err) goto fail_do_fork; child_thread = info.child_thread; child_task = info.child_task; spinlock_lock(&this_task->lock); list_add(&this_task->children, &child_task->list); spinlock_unlock(&this_task->lock); fork_dmsg(1, "%s: childs (task & thread) have been registered in their parents lists [%d]\n", __FUNCTION__, cpu_time_stamp()); fork_dmsg(1, "%s: going to add child to target scheduler\n", __FUNCTION__); sched_add_created(child_thread); tm_end = cpu_time_stamp(); fork_dmsg(1, "%s: cpu %d, pid %d, done [s:%u, bR:%u, aR:%u, e:%u, d:%u, t:%u, r:%u]\n", __FUNCTION__, cpu_get_id(), this_task->pid, tm_start, tm_bRemote, tm_aRemote, tm_end, attr.tm_request, tm_end - tm_start, info.tm_event); return child_task->pid; fail_do_fork: fail_childs_nr: atomic_add(&this_task->childs_nr, -1); this_thread->info.errno = err; return -1; }
error_t do_fork(fork_info_t *info) { kmem_req_t req; struct dqdt_attr_s attr; struct thread_s *child_thread; struct task_s *child_task; struct page_s *page; uint_t cid; error_t err; sint_t order; fork_dmsg(1, "%s: cpu %d, started [%d]\n", __FUNCTION__, cpu_get_id(), cpu_time_stamp()); child_thread = NULL; child_task = NULL; page = NULL; cid = info->cpu->cluster->id; attr.cid = cid; attr.cpu_id = 0; attr.cid_exec = info->cid_exec; //dqdt_update_threads_number(attr.cluster->levels_tbl[0], attr.cpu->lid, 1); dqdt_update_threads_number(cid, attr.cpu_id, 1); //attr.cluster = info->current_clstr; attr.cid = cid; err = task_create(&child_task, &attr, CPU_USR_MODE); //attr.cluster = info->cpu->cluster; attr.cid = cid; if(err) goto fail_task; fork_dmsg(1, "%s: cpu %d, ppid %d, task @0x%x, pid %d, task @0x%x [%d]\n", __FUNCTION__, cpu_get_id(), info->this_task->pid, info->this_task, child_task->pid, child_task, cpu_time_stamp()); req.type = KMEM_PAGE; req.size = ARCH_THREAD_PAGE_ORDER; req.flags = AF_KERNEL | AF_REMOTE; req.ptr = info->cpu->cluster; req.ptr = info->current_clstr; page = kmem_alloc(&req); if(page == NULL) goto fail_mem; fork_dmsg(1, "%s: child pid will be %d on cluster %d, cpu %d [%d]\n", __FUNCTION__, child_task->pid, child_task->cpu->cluster->id, child_task->cpu->gid, cpu_time_stamp()); err = task_dup(child_task, info->this_task); if(err) goto fail_task_dup; signal_manager_destroy(child_task); signal_manager_init(child_task); fork_dmsg(1, "%s: parent task has been duplicated [%d]\n", __FUNCTION__, cpu_time_stamp()); child_task->current_clstr = info->current_clstr; err = vmm_dup(&child_task->vmm, &info->this_task->vmm); if(err) goto fail_vmm_dup; fork_dmsg(1, "%s: parent vmm has been duplicated [%d]\n", __FUNCTION__, cpu_time_stamp()); child_thread = (struct thread_s*) ppm_page2addr(page); /* Set the child page before calling thread_dup */ child_thread->info.page = page; err = thread_dup(child_task, child_thread, info->cpu, info->cpu->cluster, info->this_thread); if(err) goto fail_thread_dup; /* Adjust child_thread attributes */ if(info->flags & PT_FORK_USE_AFFINITY) { child_thread->info.attr.flags |= (info->flags & ~(PT_ATTR_LEGACY_MASK)); if(!(info->flags & PT_ATTR_MEM_PRIO)) child_thread->info.attr.flags &= ~(PT_ATTR_MEM_PRIO); if(!(info->flags & PT_ATTR_AUTO_MGRT)) child_thread->info.attr.flags &= ~(PT_ATTR_AUTO_MGRT); if(!(info->flags & PT_ATTR_AUTO_NXTT)) child_thread->info.attr.flags &= ~(PT_ATTR_AUTO_NXTT); } fork_dmsg(1, "%s: parent current thread has been duplicated, tid %x [%d]\n", __FUNCTION__, child_thread, cpu_time_stamp()); if(info->isPinned) thread_migration_disabled(child_thread); else thread_migration_enabled(child_thread); list_add_last(&child_task->th_root, &child_thread->rope); child_task->threads_count = 1; child_task->threads_nr ++; child_task->state = TASK_READY; order = bitmap_ffs2(child_task->bitmap, 0, sizeof(child_task->bitmap)); if(order == -1) goto fail_order; bitmap_clear(child_task->bitmap, order); child_thread->info.attr.key = order; child_thread->info.order = order; child_task->next_order = order + 1; child_task->max_order = order; child_task->uid = info->this_task->uid; child_task->parent = info->this_task->pid; err = sched_register(child_thread); assert(err == 0); cpu_context_set_tid(&child_thread->info.pss, (reg_t)child_thread); cpu_context_set_pmm(&child_thread->info.pss, &child_task->vmm.pmm); cpu_context_dup_finlize(&child_thread->pws, &child_thread->info.pss); child_thread->info.retval = 0; child_thread->info.errno = 0; info->child_thread = child_thread; info->child_task = child_task; return 0; fail_order: fail_thread_dup: fail_vmm_dup: fail_task_dup: printk(WARNING, "WARNING: %s: destroy child thread\n", __FUNCTION__); req.ptr = page; kmem_free(&req); fail_mem: fail_task: //FIXME //dqdt_update_threads_number(attr.cluster->levels_tbl[0], attr.cpu->lid, -1); dqdt_update_threads_number(attr.cid, attr.cpu_id, -1); printk(WARNING, "WARNING: %s: destroy child task\n", __FUNCTION__); if(child_task != NULL) task_destroy(child_task); printk(WARNING, "WARNING: %s: fork err %d [%d]\n", __FUNCTION__, err, cpu_time_stamp()); return err; }
static void barrier_do_broadcast(struct barrier_s *barrier) { register uint_t tm_first; register uint_t tm_last; register uint_t tm_start; register uint_t wqdbsz; register uint_t tm_end; register uint_t ticket; register uint_t index; register uint_t count; register uint_t event; register void *listner; register wqdb_t *wqdb; register uint_t i; tm_start = cpu_time_stamp(); tm_first = barrier->tm_first; tm_last = barrier->tm_last; wqdbsz = PMM_PAGE_SIZE / sizeof(wqdb_record_t); ticket = 0; #if ARCH_HAS_BARRIERS count = barrier->count; #else count = barrier->count - 1; /* last don't sleep */ #endif for(index = 0; ((index < BARRIER_WQDB_NR) && (ticket < count)); index++) { wqdb = barrier->wqdb_tbl[index]; for(i = 0; ((i < wqdbsz) && (ticket < count)); i++) { #if CONFIG_BARRIER_BORADCAST_UREAD event = cpu_uncached_read(&wqdb->tbl[i].event); listner = (void*) cpu_uncached_read(&wqdb->tbl[i].listner); #else event = wqdb->tbl[i].event; listner = wqdb->tbl[i].listner; #endif if(listner != NULL) { wqdb->tbl[i].listner = NULL; #if CONFIG_USE_SCHED_LOCKS sched_wakeup((struct thread_s*) listner); #else sched_event_send(listner, event); #endif ticket ++; } } } tm_end = cpu_time_stamp(); printk(INFO, "INFO: %s: cpu %d [F: %d, L: %d, B: %d, E: %d, T: %d]\n", __FUNCTION__, cpu_get_id(), tm_first, tm_last, tm_start, tm_end, tm_end - tm_first); }
/* TODO: reintroduce barrier's ops to deal with case-specific treatment */ error_t barrier_wait(struct barrier_s *barrier) { register uint_t ticket; register uint_t index; register uint_t wqdbsz; register wqdb_t *wqdb; register bool_t isShared; struct thread_s *this; uint_t tm_now; tm_now = cpu_time_stamp(); this = current_thread; index = this->info.order; ticket = 0; isShared = (barrier->owner == NULL) ? true : false; if((barrier->signature != BARRIER_ID) || ((isShared == false) && (barrier->owner != this->task))) return EINVAL; wqdbsz = PMM_PAGE_SIZE / sizeof(wqdb_record_t); if(isShared) { spinlock_lock(&barrier->lock); index = barrier->index ++; ticket = barrier->count - index; } wqdb = barrier->wqdb_tbl[index / wqdbsz]; #if CONFIG_USE_SCHED_LOCKS wqdb->tbl[index % wqdbsz].listner = (void*)this; #else uint_t irq_state; cpu_disable_all_irq(&irq_state); /* To prevent against any scheduler intervention */ wqdb->tbl[index % wqdbsz].event = sched_event_make (this, SCHED_OP_WAKEUP); wqdb->tbl[index % wqdbsz].listner = sched_get_listner(this, SCHED_OP_WAKEUP); #endif if(isShared == false) ticket = atomic_add(&barrier->waiting, -1); if(ticket == 1) { #if !(CONFIG_USE_SCHED_LOCKS) cpu_restore_irq(irq_state); #endif barrier->tm_last = tm_now; wqdb->tbl[index % wqdbsz].listner = NULL; if(isShared) { barrier->index = 0; spinlock_unlock(&barrier->lock); } else atomic_init(&barrier->waiting, barrier->count); barrier_do_broadcast(barrier); return PTHREAD_BARRIER_SERIAL_THREAD; } if(ticket == barrier->count) barrier->tm_first = tm_now; spinlock_unlock_nosched(&barrier->lock); sched_sleep(this); #if !(CONFIG_USE_SCHED_LOCKS) cpu_restore_irq(irq_state); #endif return 0; }
void* kvfsd(void *arg) { uint_t tm_now, cntr; struct task_s *task; struct thread_s *this; struct cpu_s *cpu; struct alarm_info_s info; struct event_s event; uint_t fs_type; error_t err; cpu_enable_all_irq(NULL); printk(INFO, "INFO: Starting KVFSD on CPU %d [ %d ]\n", cpu_get_id(), cpu_time_stamp()); task = current_task; fs_type = VFS_TYPES_NR; #if CONFIG_ROOTFS_IS_EXT2 fs_type = VFS_EXT2_TYPE; #endif #if CONFIG_ROOTFS_IS_VFAT #if CONFIG_ROOTFS_IS_EXT2 #error More than one root fs has been selected #endif fs_type = VFS_VFAT_TYPE; #endif /* CONFIG_ROOTFS_IS_VFAT_TYPE */ err = vfs_init(__sys_blk, fs_type, VFS_MAX_NODE_NUMBER, VFS_MAX_FILE_NUMBER, &task->vfs_root); task->vfs_cwd = task->vfs_root; printk(INFO, "INFO: Virtual File System (VFS) Is Ready\n"); sysconf_init(); if(err == 0) { if((err = task_load_init(task))) { printk(WARNING, "WARNING: failed to load user process, err %d [%u]\n", err, cpu_time_stamp()); } } #if CONFIG_DEV_VERSION if(err != 0) { struct thread_s *thread; printk(INFO, "INFO: Creating kernel level terminal\n"); thread = kthread_create(task, &kMiniShelld, NULL, current_cluster->id, current_cpu->lid); thread->task = task; list_add_last(&task->th_root, &thread->rope); err = sched_register(thread); assert(err == 0); sched_add_created(thread); } #endif this = current_thread; cpu = current_cpu; event_set_senderId(&event, this); event_set_priority(&event, E_FUNC); event_set_handler(&event, &kvfsd_alarm_event_handler); info.event = &event; cntr = 0; while(1) { alarm_wait(&info, 10); sched_sleep(this); tm_now = cpu_time_stamp(); printk(INFO, "INFO: System Current TimeStamp %u\n", tm_now); sync_all_pages(); if((cntr % 4) == 0) dqdt_print_summary(dqdt_root); cntr ++; } return NULL; }
/* * FIXME: define spinlock_rdlock() so all locking on task->th_lock * becoms rdlock but on join/detach/destroy */ int sys_thread_wakeup(pthread_t tid, pthread_t *tid_tbl, uint_t count) { struct task_s *task; struct thread_s *this; struct thread_s *target; pthread_t tbl[100]; void *listner; uint_t event; sint_t i; error_t err; this = current_thread; task = this->task; i = -1; if(tid_tbl != NULL) { if((((uint_t)tid_tbl + (count*sizeof(pthread_t))) >= CONFIG_KERNEL_OFFSET) || (count == 0) || (count > 100)) { err = -1; goto fail_tid_tbl; } if((err = cpu_uspace_copy(&tbl[0], tid_tbl, sizeof(pthread_t*) * count))) goto fail_usapce; if(tbl[0] != tid) { err = -2; goto fail_first_tid; } } else { count = 1; tbl[0] = tid; } for(i = 0; i < count; i++) { tid = tbl[i]; if(tid > task->max_order) { err = -3; goto fail_tid; } target = task->th_tbl[tid]; if((target == NULL) || (target->signature != THREAD_ID)) { err = -4; goto fail_target; } listner = sched_get_listner(target, SCHED_OP_UWAKEUP); event = sched_event_make(target,SCHED_OP_UWAKEUP); if(this->info.isTraced == true) { printk(INFO,"%s: tid %d --> tid %d [%d][%d]\n", __FUNCTION__, this->info.order, tid, cpu_time_stamp(), i); } sched_event_send(listner,event); cpu_wbflush(); } return 0; fail_target: fail_tid: fail_first_tid: fail_usapce: fail_tid_tbl: printk(INFO, "%s: cpu %d, pid %d, tid %x, i %d, count %d, ttid %x, request has failed with err %d [%d]\n", __FUNCTION__, cpu_get_id(), task->pid, this, i, count, tid, err, cpu_time_stamp()); this->info.errno = EINVAL; return -1; }