void update_vsyscall_old(struct timespec *wall, struct timespec *wtm, struct clocksource *c, u32 mult) { write_seqcount_begin(&fsyscall_gtod_data.seq); /* copy fsyscall clock data */ fsyscall_gtod_data.clk_mask = c->mask; fsyscall_gtod_data.clk_mult = mult; fsyscall_gtod_data.clk_shift = c->shift; fsyscall_gtod_data.clk_fsys_mmio = c->archdata.fsys_mmio; fsyscall_gtod_data.clk_cycle_last = c->cycle_last; /* copy kernel time structures */ fsyscall_gtod_data.wall_time.tv_sec = wall->tv_sec; fsyscall_gtod_data.wall_time.tv_nsec = wall->tv_nsec; fsyscall_gtod_data.monotonic_time.tv_sec = wtm->tv_sec + wall->tv_sec; fsyscall_gtod_data.monotonic_time.tv_nsec = wtm->tv_nsec + wall->tv_nsec; /* normalize */ while (fsyscall_gtod_data.monotonic_time.tv_nsec >= NSEC_PER_SEC) { fsyscall_gtod_data.monotonic_time.tv_nsec -= NSEC_PER_SEC; fsyscall_gtod_data.monotonic_time.tv_sec++; } write_seqcount_end(&fsyscall_gtod_data.seq); }
void update_vsyscall(struct timekeeper *tk) { write_seqcount_begin(&fsyscall_gtod_data.seq); /* copy vsyscall data */ fsyscall_gtod_data.clk_mask = tk->tkr_mono.mask; fsyscall_gtod_data.clk_mult = tk->tkr_mono.mult; fsyscall_gtod_data.clk_shift = tk->tkr_mono.shift; fsyscall_gtod_data.clk_fsys_mmio = tk->tkr_mono.clock->archdata.fsys_mmio; fsyscall_gtod_data.clk_cycle_last = tk->tkr_mono.cycle_last; fsyscall_gtod_data.wall_time.sec = tk->xtime_sec; fsyscall_gtod_data.wall_time.snsec = tk->tkr_mono.xtime_nsec; fsyscall_gtod_data.monotonic_time.sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; fsyscall_gtod_data.monotonic_time.snsec = tk->tkr_mono.xtime_nsec + ((u64)tk->wall_to_monotonic.tv_nsec << tk->tkr_mono.shift); /* normalize */ while (fsyscall_gtod_data.monotonic_time.snsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { fsyscall_gtod_data.monotonic_time.snsec -= ((u64)NSEC_PER_SEC) << tk->tkr_mono.shift; fsyscall_gtod_data.monotonic_time.sec++; } write_seqcount_end(&fsyscall_gtod_data.seq); }
static void est_timer(unsigned long arg) { struct net_rate_estimator *est = (struct net_rate_estimator *)arg; struct gnet_stats_basic_packed b; u64 rate, brate; est_fetch_counters(est, &b); brate = (b.bytes - est->last_bytes) << (8 - est->ewma_log); brate -= (est->avbps >> est->ewma_log); rate = (u64)(b.packets - est->last_packets) << (8 - est->ewma_log); rate -= (est->avpps >> est->ewma_log); write_seqcount_begin(&est->seq); est->avbps += brate; est->avpps += rate; write_seqcount_end(&est->seq); est->last_bytes = b.bytes; est->last_packets = b.packets; est->next_jiffies += ((HZ/4) << est->intvl_log); if (unlikely(time_after_eq(jiffies, est->next_jiffies))) { /* Ouch... timer was delayed. */ est->next_jiffies = jiffies + 1; } mod_timer(&est->timer, est->next_jiffies); }
void chroot_fs_refs(struct path *old_root, struct path *new_root) { struct task_struct *g, *p; struct fs_struct *fs; int count = 0; read_lock(&tasklist_lock); do_each_thread(g, p) { task_lock(p); fs = p->fs; if (fs) { spin_lock(&fs->lock); write_seqcount_begin(&fs->seq); if (fs->root.dentry == old_root->dentry && fs->root.mnt == old_root->mnt) { path_get_longterm(new_root); fs->root = *new_root; count++; } if (fs->pwd.dentry == old_root->dentry && fs->pwd.mnt == old_root->mnt) { path_get_longterm(new_root); fs->pwd = *new_root; count++; } write_seqcount_end(&fs->seq); spin_unlock(&fs->lock); } task_unlock(p); } while_each_thread(g, p);
void reservation_object_add_excl_fence(struct reservation_object *obj, struct fence *fence) { struct fence *old_fence = reservation_object_get_excl(obj); struct reservation_object_list *old; u32 i = 0; old = reservation_object_get_list(obj); if (old) i = old->shared_count; if (fence) fence_get(fence); preempt_disable(); write_seqcount_begin(&obj->seq); /* write_seqcount_begin provides the necessary memory barrier */ RCU_INIT_POINTER(obj->fence_excl, fence); if (old) old->shared_count = 0; write_seqcount_end(&obj->seq); preempt_enable(); /* inplace update, no shared fences */ while (i--) fence_put(rcu_dereference_protected(old->shared[i], reservation_object_held(obj))); if (old_fence) fence_put(old_fence); }
void enabled_wait(void) { struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); unsigned long long idle_time; unsigned long psw_mask; trace_hardirqs_on(); /* Wait for external, I/O or machine check interrupt. */ psw_mask = PSW_KERNEL_BITS | PSW_MASK_WAIT | PSW_MASK_DAT | PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; clear_cpu_flag(CIF_NOHZ_DELAY); /* Call the assembler magic in entry.S */ psw_idle(idle, psw_mask); trace_hardirqs_off(); /* Account time spent with enabled wait psw loaded as idle time. */ write_seqcount_begin(&idle->seqcount); idle_time = idle->clock_idle_exit - idle->clock_idle_enter; idle->clock_idle_enter = idle->clock_idle_exit = 0ULL; idle->idle_time += idle_time; idle->idle_count++; account_idle_time(idle_time); write_seqcount_end(&idle->seqcount); }
void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, struct clocksource *clock, u32 mult) { struct timespec monotonic; write_seqcount_begin(&vsyscall_gtod_data.seq); /* copy vsyscall data */ vsyscall_gtod_data.clock.vclock_mode = clock->archdata.vclock_mode; vsyscall_gtod_data.clock.cycle_last = clock->cycle_last; vsyscall_gtod_data.clock.mask = clock->mask; vsyscall_gtod_data.clock.mult = mult; vsyscall_gtod_data.clock.shift = clock->shift; vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec; vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec; monotonic = timespec_add(*wall_time, *wtm); vsyscall_gtod_data.monotonic_time_sec = monotonic.tv_sec; vsyscall_gtod_data.monotonic_time_nsec = monotonic.tv_nsec; vsyscall_gtod_data.wall_time_coarse = __current_kernel_time(); vsyscall_gtod_data.monotonic_time_coarse = timespec_add(vsyscall_gtod_data.wall_time_coarse, *wtm); write_seqcount_end(&vsyscall_gtod_data.seq); }
static void reservation_object_add_shared_replace(struct reservation_object *obj, struct reservation_object_list *old, struct reservation_object_list *fobj, struct fence *fence) { unsigned i; struct fence *old_fence = NULL; fence_get(fence); if (!old) { RCU_INIT_POINTER(fobj->shared[0], fence); fobj->shared_count = 1; goto done; } /* * no need to bump fence refcounts, rcu_read access * requires the use of kref_get_unless_zero, and the * references from the old struct are carried over to * the new. */ fobj->shared_count = old->shared_count; for (i = 0; i < old->shared_count; ++i) { struct fence *check; check = rcu_dereference_protected(old->shared[i], reservation_object_held(obj)); if (!old_fence && check->context == fence->context) { old_fence = check; RCU_INIT_POINTER(fobj->shared[i], fence); } else RCU_INIT_POINTER(fobj->shared[i], check); } if (!old_fence) { RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); fobj->shared_count++; } done: preempt_disable(); write_seqcount_begin(&obj->seq); /* * RCU_INIT_POINTER can be used here, * seqcount provides the necessary barriers */ RCU_INIT_POINTER(obj->fence, fobj); write_seqcount_end(&obj->seq); preempt_enable(); if (old) kfree_rcu(old, rcu); if (old_fence) fence_put(old_fence); }
// ARM10C 20160521 // current->fs: (&init_task)->fs, &root void set_fs_pwd(struct fs_struct *fs, const struct path *path) { struct path old_pwd; // path: &root path_get(path); // path_get에서 한일: // [pcp0] (kmem_cache#2-oX (struct mount))->mnt_pcp->mnt_count 을 1만큼 증가 시킴 // (&(kmem_cache#5-oX (struct dentry))->d_lockref)->count: 1 만큼 증가 시킴 // &fs->lock: &((&init_task)->fs)->lock spin_lock(&fs->lock); // spin_lock 에서 한일: // &((&init_task)->fs)->lock 을 사용하여 spin lock 을 수행 // &fs->seq: &((&init_task)->fs)->seq write_seqcount_begin(&fs->seq); // write_seqcount_begin에서 한일: // (&((&init_task)->fs)->seq)->sequence: 1 // 공유자원을 다른 cpu core가 사용할수 있게 메모리 적용 // fs->pwd: ((&init_task)->fs)->pwd: (&init_fs)->pwd: 맴버가 0 으로 초기화된 값 old_pwd = fs->pwd; // old_pwd: 맴버가 0 으로 초기화된 값 // root.mnt: &(kmem_cache#2-oX (struct mount))->mnt // root.dentry: kmem_cache#5-oX (struct dentry) // fs->pwd: ((&init_task)->fs)->pwd: (&init_fs)->pwd: 맴버가 0 으로 초기화된 값, *path: root fs->pwd = *path; // fs->pwd: ((&init_task)->fs)->pwd.mnt: &(kmem_cache#2-oX (struct mount))->mnt // fs->pwd: ((&init_task)->fs)->pwd.dentry: kmem_cache#5-oX (struct dentry) // &fs->seq: &((&init_task)->fs)->seq write_seqcount_end(&fs->seq); // write_seqcount_end에서 한일: // 공유자원을 다른 cpu core가 사용할수 있게 메모리 적용 // (&((&init_task)->fs)->seq)->sequence: 2 // &fs->lock: &((&init_task)->fs)->lock spin_unlock(&fs->lock); // spin_unlock 에서 한일: // &((&init_task)->fs)->lock 을 사용하여 spin unlock 을 수행 // old_pwd.dentry: NULL if (old_pwd.dentry) path_put(&old_pwd); }
static void reservation_object_add_shared_inplace(struct reservation_object *obj, struct reservation_object_list *fobj, struct fence *fence) { u32 i; fence_get(fence); preempt_disable(); write_seqcount_begin(&obj->seq); for (i = 0; i < fobj->shared_count; ++i) { struct fence *old_fence; old_fence = rcu_dereference_protected(fobj->shared[i], reservation_object_held(obj)); if (old_fence->context == fence->context) { /* memory barrier is added by write_seqcount_begin */ RCU_INIT_POINTER(fobj->shared[i], fence); write_seqcount_end(&obj->seq); preempt_enable(); fence_put(old_fence); return; } } /* * memory barrier is added by write_seqcount_begin, * fobj->shared_count is protected by this lock too */ RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); fobj->shared_count++; write_seqcount_end(&obj->seq); preempt_enable(); }
/* * Initialize and return retrieve the jiffies update. */ static ktime_t tick_init_jiffy_update(void) { ktime_t period; raw_spin_lock(&xtime_lock); write_seqcount_begin(&xtime_seq); /* Did we start the jiffies update yet ? */ if (last_jiffies_update.tv64 == 0) last_jiffies_update = tick_next_period; period = last_jiffies_update; write_seqcount_end(&xtime_seq); raw_spin_unlock(&xtime_lock); return period; }
void psi_memstall_tick(struct task_struct *task, int cpu) { struct psi_group *group; void *iter = NULL; while ((group = iterate_groups(task, &iter))) { struct psi_group_cpu *groupc; groupc = per_cpu_ptr(group->pcpu, cpu); write_seqcount_begin(&groupc->seq); record_times(groupc, cpu, true); write_seqcount_end(&groupc->seq); } }
/* * Replace the fs->{rootmnt,root} with {mnt,dentry}. Put the old values. * It can block. */ void set_fs_root(struct fs_struct *fs, struct path *path) { struct path old_root; spin_lock(&fs->lock); write_seqcount_begin(&fs->seq); old_root = fs->root; fs->root = *path; path_get_longterm(path); write_seqcount_end(&fs->seq); spin_unlock(&fs->lock); if (old_root.dentry) path_put_longterm(&old_root); }
/* * Replace the fs->{pwdmnt,pwd} with {mnt,dentry}. Put the old values. * It can block. */ void set_fs_pwd(struct fs_struct *fs, struct path *path) { struct path old_pwd; spin_lock(&fs->lock); write_seqcount_begin(&fs->seq); old_pwd = fs->pwd; fs->pwd = *path; path_get_longterm(path); write_seqcount_end(&fs->seq); spin_unlock(&fs->lock); if (old_pwd.dentry) path_put_longterm(&old_pwd); }
/* * Periodic tick */ static void tick_periodic(int cpu) { if (tick_do_timer_cpu == cpu) { raw_spin_lock(&xtime_lock); write_seqcount_begin(&xtime_seq); /* Keep track of the next tick event */ tick_next_period = ktime_add(tick_next_period, tick_period); do_timer(1); write_seqcount_end(&xtime_seq); raw_spin_unlock(&xtime_lock); } update_process_times(user_mode(get_irq_regs())); profile_tick(CPU_PROFILING); }
static inline void nft_counter_do_eval(struct nft_counter_percpu_priv *priv, struct nft_regs *regs, const struct nft_pktinfo *pkt) { struct nft_counter *this_cpu; seqcount_t *myseq; local_bh_disable(); this_cpu = this_cpu_ptr(priv->counter); myseq = this_cpu_ptr(&nft_counter_seq); write_seqcount_begin(myseq); this_cpu->bytes += pkt->skb->len; this_cpu->packets++; write_seqcount_end(myseq); local_bh_enable(); }
/* * Must be called with interrupts disabled ! */ static void tick_do_update_jiffies64(ktime_t now) { unsigned long ticks = 0; ktime_t delta; /* * Do a quick check without holding xtime_lock: */ delta = ktime_sub(now, last_jiffies_update); if (delta.tv64 < tick_period.tv64) return; /* Reevalute with xtime_lock held */ raw_spin_lock(&xtime_lock); write_seqcount_begin(&xtime_seq); delta = ktime_sub(now, last_jiffies_update); if (delta.tv64 >= tick_period.tv64) { delta = ktime_sub(delta, tick_period); last_jiffies_update = ktime_add(last_jiffies_update, tick_period); /* Slow path for long timeouts */ if (unlikely(delta.tv64 >= tick_period.tv64)) { s64 incr = ktime_to_ns(tick_period); ticks = ktime_divns(delta, incr); last_jiffies_update = ktime_add_ns(last_jiffies_update, incr * ticks); } do_timer(++ticks); /* Keep the tick_next_period variable up to date */ tick_next_period = ktime_add(last_jiffies_update, tick_period); } write_seqcount_end(&xtime_seq); raw_spin_unlock(&xtime_lock); }
static void psi_group_change(struct psi_group *group, int cpu, unsigned int clear, unsigned int set) { struct psi_group_cpu *groupc; unsigned int t, m; groupc = per_cpu_ptr(group->pcpu, cpu); /* * First we assess the aggregate resource states this CPU's * tasks have been in since the last change, and account any * SOME and FULL time these may have resulted in. * * Then we update the task counts according to the state * change requested through the @clear and @set bits. */ write_seqcount_begin(&groupc->seq); record_times(groupc, cpu, false); for (t = 0, m = clear; m; m &= ~(1 << t), t++) { if (!(m & (1 << t))) continue; if (groupc->tasks[t] == 0 && !psi_bug) { printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u] clear=%x set=%x\n", cpu, t, groupc->tasks[0], groupc->tasks[1], groupc->tasks[2], clear, set); psi_bug = 1; } groupc->tasks[t]--; } for (t = 0; set; set &= ~(1 << t), t++) if (set & (1 << t)) groupc->tasks[t]++; write_seqcount_end(&groupc->seq); }