From f55db609042faecd5e518ce372b87f846659b32e Mon Sep 17 00:00:00 2001 From: Stanislaw Gruszka Date: Thu, 11 Mar 2010 14:04:37 -0800 Subject: cpu-timers: Simplify RLIMIT_CPU handling Let always set signal->cputime_expires expiration cache when setting new itimer, POSIX 1.b timer, and RLIMIT_CPU. Since we are initializing prof_exp expiration cache during fork(), this allows to remove "RLIMIT_CPU != inf" check from fastpath_timer_check() and do some other cleanups. Checked against regression using test cases from: http://marc.info/?l=linux-kernel&m=123749066504641&w=4 http://marc.info/?l=linux-kernel&m=123811277916642&w=2 Signed-off-by: Stanislaw Gruszka Cc: Ingo Molnar Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Hidetoshi Seto Cc: Balbir Singh Signed-off-by: Andrew Morton Signed-off-by: Thomas Gleixner --- kernel/posix-cpu-timers.c | 75 +++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 48 deletions(-) diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 1a22dfd42df..d01e0a348e6 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c @@ -11,19 +11,18 @@ #include /* - * Called after updating RLIMIT_CPU to set timer expiration if necessary. + * Called after updating RLIMIT_CPU to run cpu timer and update + * tsk->signal->cputime_expires expiration cache if necessary. Needs + * siglock protection since other code may update expiration cache as + * well. */ void update_rlimit_cpu(unsigned long rlim_new) { cputime_t cputime = secs_to_cputime(rlim_new); - struct signal_struct *const sig = current->signal; - if (cputime_eq(sig->it[CPUCLOCK_PROF].expires, cputime_zero) || - cputime_gt(sig->it[CPUCLOCK_PROF].expires, cputime)) { - spin_lock_irq(¤t->sighand->siglock); - set_process_cpu_timer(current, CPUCLOCK_PROF, &cputime, NULL); - spin_unlock_irq(¤t->sighand->siglock); - } + spin_lock_irq(¤t->sighand->siglock); + set_process_cpu_timer(current, CPUCLOCK_PROF, &cputime, NULL); + spin_unlock_irq(¤t->sighand->siglock); } static int check_clock(const clockid_t which_clock) @@ -564,7 +563,6 @@ static void arm_timer(struct k_itimer *timer, union cpu_time_count now) struct list_head *head, *listpos; struct cpu_timer_list *const nt = &timer->it.cpu; struct cpu_timer_list *next; - unsigned long i; head = (CPUCLOCK_PERTHREAD(timer->it_clock) ? p->cpu_timers : p->signal->cpu_timers); @@ -630,20 +628,11 @@ static void arm_timer(struct k_itimer *timer, union cpu_time_count now) default: BUG(); case CPUCLOCK_VIRT: - if (expires_le(sig->it[CPUCLOCK_VIRT].expires, - exp->cpu)) - break; - sig->cputime_expires.virt_exp = exp->cpu; - break; + if (expires_gt(sig->cputime_expires.virt_exp, exp->cpu)) + sig->cputime_expires.virt_exp = exp->cpu; case CPUCLOCK_PROF: - if (expires_le(sig->it[CPUCLOCK_PROF].expires, - exp->cpu)) - break; - i = sig->rlim[RLIMIT_CPU].rlim_cur; - if (i != RLIM_INFINITY && - i <= cputime_to_secs(exp->cpu)) - break; - sig->cputime_expires.prof_exp = exp->cpu; + if (expires_gt(sig->cputime_expires.prof_exp, exp->cpu)) + sig->cputime_expires.prof_exp = exp->cpu; break; case CPUCLOCK_SCHED: sig->cputime_expires.sched_exp = exp->sched; @@ -1386,7 +1375,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk) return 1; } - return sig->rlim[RLIMIT_CPU].rlim_cur != RLIM_INFINITY; + return 0; } /* @@ -1452,21 +1441,23 @@ void run_posix_cpu_timers(struct task_struct *tsk) } /* - * Set one of the process-wide special case CPU timers. + * Set one of the process-wide special case CPU timers or RLIMIT_CPU. * The tsk->sighand->siglock must be held by the caller. - * The *newval argument is relative and we update it to be absolute, *oldval - * is absolute and we update it to be relative. */ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, cputime_t *newval, cputime_t *oldval) { union cpu_time_count now; - struct list_head *head; BUG_ON(clock_idx == CPUCLOCK_SCHED); cpu_timer_sample_group(clock_idx, tsk, &now); if (oldval) { + /* + * We are setting itimer. The *oldval is absolute and we update + * it to be relative, *newval argument is relative and we update + * it to be absolute. + */ if (!cputime_eq(*oldval, cputime_zero)) { if (cputime_le(*oldval, now.cpu)) { /* Just about to fire. */ @@ -1479,33 +1470,21 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, if (cputime_eq(*newval, cputime_zero)) return; *newval = cputime_add(*newval, now.cpu); - - /* - * If the RLIMIT_CPU timer will expire before the - * ITIMER_PROF timer, we have nothing else to do. - */ - if (tsk->signal->rlim[RLIMIT_CPU].rlim_cur - < cputime_to_secs(*newval)) - return; } /* - * Check whether there are any process timers already set to fire - * before this one. If so, we don't have anything more to do. + * Update expiration cache if we are the earliest timer, or eventually + * RLIMIT_CPU limit is earlier than prof_exp cpu timer expire. */ - head = &tsk->signal->cpu_timers[clock_idx]; - if (list_empty(head) || - cputime_ge(list_first_entry(head, - struct cpu_timer_list, entry)->expires.cpu, - *newval)) { - switch (clock_idx) { - case CPUCLOCK_PROF: + switch (clock_idx) { + case CPUCLOCK_PROF: + if (expires_gt(tsk->signal->cputime_expires.prof_exp, *newval)) tsk->signal->cputime_expires.prof_exp = *newval; - break; - case CPUCLOCK_VIRT: + break; + case CPUCLOCK_VIRT: + if (expires_gt(tsk->signal->cputime_expires.virt_exp, *newval)) tsk->signal->cputime_expires.virt_exp = *newval; - break; - } + break; } } -- cgit v1.2.3 From 5eb9aa6414bdab6d075a8763bc3b647181ef3aab Mon Sep 17 00:00:00 2001 From: Stanislaw Gruszka Date: Thu, 11 Mar 2010 14:04:38 -0800 Subject: cpu-timers: Cleanup arm_timer() Signed-off-by: Stanislaw Gruszka Cc: Ingo Molnar Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Hidetoshi Seto Cc: Balbir Singh Signed-off-by: Andrew Morton Signed-off-by: Thomas Gleixner --- kernel/posix-cpu-timers.c | 104 ++++++++++++++++------------------------------ 1 file changed, 35 insertions(+), 69 deletions(-) diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index d01e0a348e6..7c7166f766c 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c @@ -547,97 +547,63 @@ static inline int expires_gt(cputime_t expires, cputime_t new_exp) cputime_gt(expires, new_exp); } -static inline int expires_le(cputime_t expires, cputime_t new_exp) -{ - return !cputime_eq(expires, cputime_zero) && - cputime_le(expires, new_exp); -} /* * Insert the timer on the appropriate list before any timers that * expire later. This must be called with the tasklist_lock held * for reading, and interrupts disabled. */ -static void arm_timer(struct k_itimer *timer, union cpu_time_count now) +static void arm_timer(struct k_itimer *timer) { struct task_struct *p = timer->it.cpu.task; struct list_head *head, *listpos; + struct task_cputime *cputime_expires; struct cpu_timer_list *const nt = &timer->it.cpu; struct cpu_timer_list *next; - head = (CPUCLOCK_PERTHREAD(timer->it_clock) ? - p->cpu_timers : p->signal->cpu_timers); + if (CPUCLOCK_PERTHREAD(timer->it_clock)) { + head = p->cpu_timers; + cputime_expires = &p->cputime_expires; + } else { + head = p->signal->cpu_timers; + cputime_expires = &p->signal->cputime_expires; + } head += CPUCLOCK_WHICH(timer->it_clock); BUG_ON(!irqs_disabled()); spin_lock(&p->sighand->siglock); listpos = head; - if (CPUCLOCK_WHICH(timer->it_clock) == CPUCLOCK_SCHED) { - list_for_each_entry(next, head, entry) { - if (next->expires.sched > nt->expires.sched) - break; - listpos = &next->entry; - } - } else { - list_for_each_entry(next, head, entry) { - if (cputime_gt(next->expires.cpu, nt->expires.cpu)) - break; - listpos = &next->entry; - } + list_for_each_entry(next, head, entry) { + if (cpu_time_before(timer->it_clock, nt->expires, next->expires)) + break; + listpos = &next->entry; } list_add(&nt->entry, listpos); if (listpos == head) { + union cpu_time_count *exp = &nt->expires; + /* - * We are the new earliest-expiring timer. - * If we are a thread timer, there can always - * be a process timer telling us to stop earlier. + * We are the new earliest-expiring POSIX 1.b timer, hence + * need to update expiration cache. Take into account that + * for process timers we share expiration cache with itimers + * and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME. */ - if (CPUCLOCK_PERTHREAD(timer->it_clock)) { - union cpu_time_count *exp = &nt->expires; - - switch (CPUCLOCK_WHICH(timer->it_clock)) { - default: - BUG(); - case CPUCLOCK_PROF: - if (expires_gt(p->cputime_expires.prof_exp, - exp->cpu)) - p->cputime_expires.prof_exp = exp->cpu; - break; - case CPUCLOCK_VIRT: - if (expires_gt(p->cputime_expires.virt_exp, - exp->cpu)) - p->cputime_expires.virt_exp = exp->cpu; - break; - case CPUCLOCK_SCHED: - if (p->cputime_expires.sched_exp == 0 || - p->cputime_expires.sched_exp > exp->sched) - p->cputime_expires.sched_exp = - exp->sched; - break; - } - } else { - struct signal_struct *const sig = p->signal; - union cpu_time_count *exp = &timer->it.cpu.expires; - - /* - * For a process timer, set the cached expiration time. - */ - switch (CPUCLOCK_WHICH(timer->it_clock)) { - default: - BUG(); - case CPUCLOCK_VIRT: - if (expires_gt(sig->cputime_expires.virt_exp, exp->cpu)) - sig->cputime_expires.virt_exp = exp->cpu; - case CPUCLOCK_PROF: - if (expires_gt(sig->cputime_expires.prof_exp, exp->cpu)) - sig->cputime_expires.prof_exp = exp->cpu; - break; - case CPUCLOCK_SCHED: - sig->cputime_expires.sched_exp = exp->sched; - break; - } + switch (CPUCLOCK_WHICH(timer->it_clock)) { + case CPUCLOCK_PROF: + if (expires_gt(cputime_expires->prof_exp, exp->cpu)) + cputime_expires->prof_exp = exp->cpu; + break; + case CPUCLOCK_VIRT: + if (expires_gt(cputime_expires->virt_exp, exp->cpu)) + cputime_expires->virt_exp = exp->cpu; + break; + case CPUCLOCK_SCHED: + if (cputime_expires->sched_exp == 0 || + cputime_expires->sched_exp > exp->sched) + cputime_expires->sched_exp = exp->sched; + break; } } @@ -819,7 +785,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, if (new_expires.sched != 0 && (timer->it_sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE && cpu_time_before(timer->it_clock, val, new_expires)) { - arm_timer(timer, val); + arm_timer(timer); } read_unlock(&tasklist_lock); @@ -1283,7 +1249,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) /* * Now re-arm for the new expiry time. */ - arm_timer(timer, now); + arm_timer(timer); out_unlock: read_unlock(&tasklist_lock); -- cgit v1.2.3 From ae1a78eecc45fe41215d9dbfd7079999455772d6 Mon Sep 17 00:00:00 2001 From: Stanislaw Gruszka Date: Thu, 11 Mar 2010 14:04:39 -0800 Subject: cpu-timers: Return correct previous timer reload value According POSIX we need to correctly set old timer it_interval value when user request that in timer_settime(). Tested using below program. #include #include #include #include #include #include #include int main(void) { struct sigaction act; struct sigevent evt = { }; timer_t tid; struct itimerspec spec, u_spec, k_spec; evt.sigev_notify = SIGEV_SIGNAL; evt.sigev_signo = SIGPROF; assert(timer_create(CLOCK_PROCESS_CPUTIME_ID, &evt, &tid) == 0); spec.it_value.tv_sec = 1; spec.it_value.tv_nsec = 2; spec.it_interval.tv_sec = 3; spec.it_interval.tv_nsec = 4; u_spec = spec; assert(timer_settime(tid, 0, &spec, NULL) == 0); spec.it_value.tv_sec = 5; spec.it_value.tv_nsec = 6; spec.it_interval.tv_sec = 7; spec.it_interval.tv_nsec = 8; assert(timer_settime(tid, 0, &spec, &k_spec) == 0); #define PRT(val) printf(#val ":\t%d/%d\n", (int) u_spec.val, (int) k_spec.val) PRT(it_value.tv_sec); PRT(it_value.tv_nsec); PRT(it_interval.tv_sec); PRT(it_interval.tv_nsec); return 0; } Signed-off-by: Stanislaw Gruszka Cc: Ingo Molnar Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Hidetoshi Seto Cc: Balbir Singh Signed-off-by: Andrew Morton Signed-off-by: Thomas Gleixner --- kernel/posix-cpu-timers.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 7c7166f766c..cce2f0b2d40 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c @@ -676,7 +676,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, struct itimerspec *new, struct itimerspec *old) { struct task_struct *p = timer->it.cpu.task; - union cpu_time_count old_expires, new_expires, val; + union cpu_time_count old_expires, new_expires, old_incr, val; int ret; if (unlikely(p == NULL)) { @@ -707,6 +707,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, BUG_ON(!irqs_disabled()); ret = 0; + old_incr = timer->it.cpu.incr; spin_lock(&p->sighand->siglock); old_expires = timer->it.cpu.expires; if (unlikely(timer->it.cpu.firing)) { @@ -822,7 +823,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, out: if (old) { sample_to_timespec(timer->it_clock, - timer->it.cpu.incr, &old->it_interval); + old_incr, &old->it_interval); } return ret; } -- cgit v1.2.3 From 1f169f84d25a74fb2dc67274d31d082ce30c60fb Mon Sep 17 00:00:00 2001 From: Stanislaw Gruszka Date: Thu, 11 Mar 2010 14:04:41 -0800 Subject: cpu-timers: Change SIGEV_NONE timer implementation When user sets up a timer without associated signal and process does not use any other cpu timers and does not exit, tsk->signal->cputimer is enabled and running forever. Avoid running the timer for no reason. I used below program to check patch does not break current user space visible behavior. #include #include #include #include #include #include #include #include void consume_cpu(void) { int i = 0; int count = 0; for(i=0; i<100000000; i++) count++; } int main(void) { int i; struct sigaction act; struct sigevent evt = { }; timer_t tid; struct itimerspec spec = { }; evt.sigev_notify = SIGEV_NONE; assert(timer_create(CLOCK_PROCESS_CPUTIME_ID, &evt, &tid) == 0); spec.it_value.tv_sec = 10; assert(timer_settime(tid, 0, &spec, NULL) == 0); for (i = 0; i < 30; i++) { consume_cpu(); memset(&spec, 0, sizeof(spec)); assert(timer_gettime(tid, &spec) == 0); printf("%lu.%09lu\n", (unsigned long) spec.it_value.tv_sec, (unsigned long) spec.it_value.tv_nsec); } assert(timer_delete(tid) == 0); return 0; } Signed-off-by: Stanislaw Gruszka Cc: Ingo Molnar Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Hidetoshi Seto Cc: Balbir Singh Signed-off-by: Andrew Morton Signed-off-by: Thomas Gleixner --- kernel/posix-cpu-timers.c | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index cce2f0b2d40..7d9d0fab165 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c @@ -615,7 +615,12 @@ static void arm_timer(struct k_itimer *timer) */ static void cpu_timer_fire(struct k_itimer *timer) { - if (unlikely(timer->sigq == NULL)) { + if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) { + /* + * User don't want any signal. + */ + timer->it.cpu.expires.sched = 0; + } else if (unlikely(timer->sigq == NULL)) { /* * This a special case for clock_nanosleep, * not a normal timer from sys_timer_create. @@ -784,7 +789,6 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, */ timer->it.cpu.expires = new_expires; if (new_expires.sched != 0 && - (timer->it_sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE && cpu_time_before(timer->it_clock, val, new_expires)) { arm_timer(timer); } @@ -809,7 +813,6 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, timer->it_overrun = -1; if (new_expires.sched != 0 && - (timer->it_sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE && !cpu_time_before(timer->it_clock, val, new_expires)) { /* * The designated time already passed, so we notify @@ -883,25 +886,6 @@ void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) read_unlock(&tasklist_lock); } - if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) { - if (timer->it.cpu.incr.sched == 0 && - cpu_time_before(timer->it_clock, - timer->it.cpu.expires, now)) { - /* - * Do-nothing timer expired and has no reload, - * so it's as if it was never set. - */ - timer->it.cpu.expires.sched = 0; - itp->it_value.tv_sec = itp->it_value.tv_nsec = 0; - return; - } - /* - * Account for any expirations and reloads that should - * have happened. - */ - bump_cpu_timer(timer, now); - } - if (unlikely(clear_dead)) { /* * We've noticed that the thread is dead, but -- cgit v1.2.3 From c28739375bf0d6e239b4fa939ec8372aa2c707d2 Mon Sep 17 00:00:00 2001 From: Stanislaw Gruszka Date: Thu, 11 Mar 2010 14:04:42 -0800 Subject: cpu-timers: Avoid iterating over all threads in fastpath_timer_check() Spread p->sighand->siglock locking scope to make sure that fastpath_timer_check() never iterates over all threads. Without locking there is small possibility that signal->cputimer will stop running while we write values to signal->cputime_expires. Calling thread_group_cputime() from fastpath_timer_check() is not only bad because it is slow, also it is racy with __exit_signal() which can lead to invalid signal->{s,u}time values. Signed-off-by: Stanislaw Gruszka Cc: Ingo Molnar Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Hidetoshi Seto Cc: Balbir Singh Signed-off-by: Andrew Morton Signed-off-by: Thomas Gleixner --- kernel/posix-cpu-timers.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 7d9d0fab165..564b3b0240d 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c @@ -550,7 +550,7 @@ static inline int expires_gt(cputime_t expires, cputime_t new_exp) /* * Insert the timer on the appropriate list before any timers that * expire later. This must be called with the tasklist_lock held - * for reading, and interrupts disabled. + * for reading, interrupts disabled and p->sighand->siglock taken. */ static void arm_timer(struct k_itimer *timer) { @@ -569,9 +569,6 @@ static void arm_timer(struct k_itimer *timer) } head += CPUCLOCK_WHICH(timer->it_clock); - BUG_ON(!irqs_disabled()); - spin_lock(&p->sighand->siglock); - listpos = head; list_for_each_entry(next, head, entry) { if (cpu_time_before(timer->it_clock, nt->expires, next->expires)) @@ -606,8 +603,6 @@ static void arm_timer(struct k_itimer *timer) break; } } - - spin_unlock(&p->sighand->siglock); } /* @@ -720,7 +715,6 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, ret = TIMER_RETRY; } else list_del_init(&timer->it.cpu.entry); - spin_unlock(&p->sighand->siglock); /* * We need to sample the current value to convert the new @@ -774,6 +768,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, * disable this firing since we are already reporting * it as an overrun (thanks to bump_cpu_timer above). */ + spin_unlock(&p->sighand->siglock); read_unlock(&tasklist_lock); goto out; } @@ -793,6 +788,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, arm_timer(timer); } + spin_unlock(&p->sighand->siglock); read_unlock(&tasklist_lock); /* @@ -1206,6 +1202,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) goto out; } read_lock(&tasklist_lock); /* arm_timer needs it. */ + spin_lock(&p->sighand->siglock); } else { read_lock(&tasklist_lock); if (unlikely(p->signal == NULL)) { @@ -1226,6 +1223,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) clear_dead_task(timer, now); goto out_unlock; } + spin_lock(&p->sighand->siglock); cpu_timer_sample_group(timer->it_clock, p, &now); bump_cpu_timer(timer, now); /* Leave the tasklist_lock locked for the call below. */ @@ -1234,7 +1232,9 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) /* * Now re-arm for the new expiry time. */ + BUG_ON(!irqs_disabled()); arm_timer(timer); + spin_unlock(&p->sighand->siglock); out_unlock: read_unlock(&tasklist_lock); -- cgit v1.2.3 From 64ce4c2f5252f25798117fa80a027993163d6d84 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Thu, 11 Mar 2010 14:04:47 -0800 Subject: time: Clean up warp_clock() warp_clock() currently accesses timekeeping internal state directly, which is unnecessary. Convert it to use the proper timekeeping interfaces. Signed-off-by: John Stultz Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Thomas Gleixner --- kernel/time.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/kernel/time.c b/kernel/time.c index 804798005d1..2358a3646a6 100644 --- a/kernel/time.c +++ b/kernel/time.c @@ -133,12 +133,11 @@ SYSCALL_DEFINE2(gettimeofday, struct timeval __user *, tv, */ static inline void warp_clock(void) { - write_seqlock_irq(&xtime_lock); - wall_to_monotonic.tv_sec -= sys_tz.tz_minuteswest * 60; - xtime.tv_sec += sys_tz.tz_minuteswest * 60; - update_xtime_cache(0); - write_sequnlock_irq(&xtime_lock); - clock_was_set(); + struct timespec delta, adjust; + delta.tv_sec = sys_tz.tz_minuteswest * 60; + delta.tv_nsec = 0; + adjust = timespec_add_safe(current_kernel_time(), delta); + do_settimeofday(&adjust); } /* -- cgit v1.2.3 From 06f71b922ce5a05352acd706564ca4ae1f2add0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Thu, 11 Mar 2010 14:04:46 -0800 Subject: timer: Print function name for timer callbacks modifying preemption count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A function scheduled with a timer must not exit with a different preempt count than it was entered. To make helping users running into the corresponding BUG() easier also print the name of the bad function not only its address. [ tglx: Sanitized printk ] Signed-off-by: Uwe Kleine-König Cc: johnstul@us.ibm.com Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Thomas Gleixner --- kernel/timer.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/kernel/timer.c b/kernel/timer.c index c61a7949387..f82f4bfe2d8 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1027,11 +1027,8 @@ static inline void __run_timers(struct tvec_base *base) lock_map_release(&lockdep_map); if (preempt_count != preempt_count()) { - printk(KERN_ERR "huh, entered %p " - "with preempt_count %08x, exited" - " with %08x?\n", - fn, preempt_count, - preempt_count()); + printk(KERN_ERR "timer: %pF preempt leak: %08x -> %08x\n", + fn, preempt_count, preempt_count()); BUG(); } } -- cgit v1.2.3 From 576da126a6c7364d70dfd58d0bbe43d05cf5859f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 12 Mar 2010 21:10:29 +0100 Subject: timer: Split out timer function call The ident level is starting to be annoying. More white space than actual code. Split out the timer function call into its own function. Signed-off-by: Thomas Gleixner --- kernel/timer.c | 72 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/kernel/timer.c b/kernel/timer.c index f82f4bfe2d8..45229694dc6 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -953,6 +953,41 @@ static int cascade(struct tvec_base *base, struct tvec *tv, int index) return index; } +static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long), + unsigned long data) +{ + int preempt_count = preempt_count(); + +#ifdef CONFIG_LOCKDEP + /* + * It is permissible to free the timer from inside the + * function that is called from it, this we need to take into + * account for lockdep too. To avoid bogus "held lock freed" + * warnings as well as problems when looking into + * timer->lockdep_map, make a copy and use that here. + */ + struct lockdep_map lockdep_map = timer->lockdep_map; +#endif + /* + * Couple the lock chain with the lock chain at + * del_timer_sync() by acquiring the lock_map around the fn() + * call here and in del_timer_sync(). + */ + lock_map_acquire(&lockdep_map); + + trace_timer_expire_entry(timer); + fn(data); + trace_timer_expire_exit(timer); + + lock_map_release(&lockdep_map); + + if (preempt_count != preempt_count()) { + printk(KERN_ERR "timer: %pF preempt leak: %08x -> %08x\n", + fn, preempt_count, preempt_count()); + BUG(); + } +} + #define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK) /** @@ -996,42 +1031,7 @@ static inline void __run_timers(struct tvec_base *base) detach_timer(timer, 1); spin_unlock_irq(&base->lock); - { - int preempt_count = preempt_count(); - -#ifdef CONFIG_LOCKDEP - /* - * It is permissible to free the timer from - * inside the function that is called from - * it, this we need to take into account for - * lockdep too. To avoid bogus "held lock - * freed" warnings as well as problems when - * looking into timer->lockdep_map, make a - * copy and use that here. - */ - struct lockdep_map lockdep_map = - timer->lockdep_map; -#endif - /* - * Couple the lock chain with the lock chain at - * del_timer_sync() by acquiring the lock_map - * around the fn() call here and in - * del_timer_sync(). - */ - lock_map_acquire(&lockdep_map); - - trace_timer_expire_entry(timer); - fn(data); - trace_timer_expire_exit(timer); - - lock_map_release(&lockdep_map); - - if (preempt_count != preempt_count()) { - printk(KERN_ERR "timer: %pF preempt leak: %08x -> %08x\n", - fn, preempt_count, preempt_count()); - BUG(); - } - } + call_timer_fn(timer, fn, data); spin_lock_irq(&base->lock); } } -- cgit v1.2.3 From 802702e0c2618465b813242d4dfee6a233ba0beb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 12 Mar 2010 20:13:23 +0100 Subject: timer: Try to survive timer callback preempt_count leak If a timer callback leaks preempt_count we currently assert a BUG(). That makes it unnecessarily hard to retrieve information about the problem especially on laptops and headless stations. There is a decent chance to survive the preempt_count leak by restoring the preempt_count to the value before the callback. That allows in many cases to get valuable information about the root cause of the problem. We carried that fixup in preempt-rt for years and were able to decode such wreckage quite a few times. Signed-off-by: Thomas Gleixner Cc: Linux Torvalds Cc: Andrew Morton Cc: Arjan van de Veen --- kernel/timer.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/kernel/timer.c b/kernel/timer.c index 45229694dc6..7e12e7bc7ce 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -982,9 +982,15 @@ static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long), lock_map_release(&lockdep_map); if (preempt_count != preempt_count()) { - printk(KERN_ERR "timer: %pF preempt leak: %08x -> %08x\n", - fn, preempt_count, preempt_count()); - BUG(); + WARN_ONCE(1, "timer: %pF preempt leak: %08x -> %08x\n", + fn, preempt_count, preempt_count()); + /* + * Restore the preempt count. That gives us a decent + * chance to survive and extract information. If the + * callback kept a lock held, bad luck, but not worse + * than the BUG() we had. + */ + preempt_count() = preempt_count; } } -- cgit v1.2.3 From 25268498c9e07870323aead10751b7c6e99a3a78 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 12 Mar 2010 14:56:00 -0800 Subject: time: Add xtime, wall_to_monotonic to feature-removal-schedule They aren't really features, but fair warning to out of tree driver folks who might be accessing these variables. Signed-off-by: John Stultz CC: Ingo Molnar LKML-Reference: <1268434560-2677-1-git-send-email-johnstul@us.ibm.com> Signed-off-by: Thomas Gleixner --- Documentation/feature-removal-schedule.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/feature-removal-schedule.txt b/Documentation/feature-removal-schedule.txt index a5cc0db63d7..198acdc05fd 100644 --- a/Documentation/feature-removal-schedule.txt +++ b/Documentation/feature-removal-schedule.txt @@ -582,3 +582,13 @@ Why: The paravirt mmu host support is slower than non-paravirt mmu, both Who: Avi Kivity ---------------------------- + +What: xtime, wall_to_monotonic +When: 2.6.36+ +Files: kernel/time/timekeeping.c include/linux/time.h +Why: Cleaning up timekeeping internal values. Please use + existing timekeeping accessor functions to access + the equivalent functionality. +Who: John Stultz + +---------------------------- -- cgit v1.2.3 From e1292ba164742e3a236e407148e00300b7196906 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Thu, 18 Mar 2010 20:19:27 -0700 Subject: ntp: Make time_adjust static Now that no arches are accessing time_adjust directly, make it static. Signed-off-by: John Stultz LKML-Reference: <1268968769-19209-1-git-send-email-johnstul@us.ibm.com> Signed-off-by: Thomas Gleixner --- include/linux/timex.h | 1 - kernel/time/ntp.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/include/linux/timex.h b/include/linux/timex.h index 7a082b32d8e..e2de51eedf0 100644 --- a/include/linux/timex.h +++ b/include/linux/timex.h @@ -238,7 +238,6 @@ extern int tickadj; /* amount of adjustment per tick */ * phase-lock loop variables */ extern int time_status; /* clock synchronization status bits */ -extern long time_adjust; /* The amount of adjtime left */ extern void ntp_init(void); extern void ntp_clear(void); diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 7c0f180d6e9..c63116863a8 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -69,7 +69,7 @@ static s64 time_freq; /* time at last adjustment (secs): */ static long time_reftime; -long time_adjust; +static long time_adjust; /* constant (boot-param configurable) NTP tick adjustment (upscaled) */ static s64 ntp_tick_adj; -- cgit v1.2.3 From 3d0205bd1383aa3cac93c209b7c7d03b27930195 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Thu, 18 Mar 2010 20:19:28 -0700 Subject: ntp: Remove tickadj There are zero users of tickadj. So remove it. Signed-off-by: John Stultz LKML-Reference: <1268968769-19209-2-git-send-email-johnstul@us.ibm.com> Signed-off-by: Thomas Gleixner --- include/linux/timex.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/linux/timex.h b/include/linux/timex.h index e2de51eedf0..32d852f8cbe 100644 --- a/include/linux/timex.h +++ b/include/linux/timex.h @@ -232,7 +232,6 @@ struct timex { */ extern unsigned long tick_usec; /* USER_HZ period (usec) */ extern unsigned long tick_nsec; /* ACTHZ period (nsec) */ -extern int tickadj; /* amount of adjustment per tick */ /* * phase-lock loop variables @@ -270,9 +269,6 @@ extern void second_overflow(void); extern void update_ntp_one_tick(void); extern int do_adjtimex(struct timex *); -/* Don't use! Compatibility define for existing users. */ -#define tickadj (500/HZ ? : 1) - int read_current_timer(unsigned long *timer_val); /* The clock frequency of the i8253/i8254 PIT */ -- cgit v1.2.3 From 3bbb9ec946428b96657126768f65487a48dd090c Mon Sep 17 00:00:00 2001 From: Arjan van de Ven Date: Thu, 11 Mar 2010 14:04:36 -0800 Subject: timers: Introduce the concept of timer slack for legacy timers While HR timers have had the concept of timer slack for quite some time now, the legacy timers lacked this concept, and had to make do with round_jiffies() and friends. Timer slack is important for power management; grouping timers reduces the number of wakeups which in turn reduces power consumption. This patch introduces timer slack to the legacy timers using the following pieces: * A slack field in the timer struct * An api (set_timer_slack) that callers can use to set explicit timer slack * A default slack of 0.4% of the requested delay for callers that do not set any explicit slack * Rounding code that is part of mod_timer() that tries to group timers around jiffies values every 'power of two' (so quick timers will group around every 2, but longer timers will group around every 4, 8, 16, 32 etc) Signed-off-by: Arjan van de Ven Cc: johnstul@us.ibm.com Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Thomas Gleixner --- include/linux/timer.h | 10 ++++++++- kernel/timer.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/include/linux/timer.h b/include/linux/timer.h index a2d1eb6cb3f..ea965b857a5 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -10,13 +10,19 @@ struct tvec_base; struct timer_list { + /* + * All fields that change during normal runtime grouped to the + * same cacheline + */ struct list_head entry; unsigned long expires; + struct tvec_base *base; void (*function)(unsigned long); unsigned long data; - struct tvec_base *base; + int slack; + #ifdef CONFIG_TIMER_STATS void *start_site; char start_comm[16]; @@ -165,6 +171,8 @@ extern int mod_timer(struct timer_list *timer, unsigned long expires); extern int mod_timer_pending(struct timer_list *timer, unsigned long expires); extern int mod_timer_pinned(struct timer_list *timer, unsigned long expires); +extern void set_timer_slack(struct timer_list *time, int slack_hz); + #define TIMER_NOT_PINNED 0 #define TIMER_PINNED 1 /* diff --git a/kernel/timer.c b/kernel/timer.c index 7e12e7bc7ce..49773f38c9b 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -318,6 +318,24 @@ unsigned long round_jiffies_up_relative(unsigned long j) } EXPORT_SYMBOL_GPL(round_jiffies_up_relative); +/** + * set_timer_slack - set the allowed slack for a timer + * @slack_hz: the amount of time (in jiffies) allowed for rounding + * + * Set the amount of time, in jiffies, that a certain timer has + * in terms of slack. By setting this value, the timer subsystem + * will schedule the actual timer somewhere between + * the time mod_timer() asks for, and that time plus the slack. + * + * By setting the slack to -1, a percentage of the delay is used + * instead. + */ +void set_timer_slack(struct timer_list *timer, int slack_hz) +{ + timer->slack = slack_hz; +} +EXPORT_SYMBOL_GPL(set_timer_slack); + static inline void set_running_timer(struct tvec_base *base, struct timer_list *timer) @@ -549,6 +567,7 @@ static void __init_timer(struct timer_list *timer, { timer->entry.next = NULL; timer->base = __raw_get_cpu_var(tvec_bases); + timer->slack = -1; #ifdef CONFIG_TIMER_STATS timer->start_site = NULL; timer->start_pid = -1; @@ -714,6 +733,41 @@ int mod_timer_pending(struct timer_list *timer, unsigned long expires) } EXPORT_SYMBOL(mod_timer_pending); +/* + * Decide where to put the timer while taking the slack into account + * + * Algorithm: + * 1) calculate the maximum (absolute) time + * 2) calculate the highest bit where the expires and new max are different + * 3) use this bit to make a mask + * 4) use the bitmask to round down the maximum time, so that all last + * bits are zeros + */ +static inline +unsigned long apply_slack(struct timer_list *timer, unsigned long expires) +{ + unsigned long expires_limit, mask; + int bit; + + expires_limit = expires + timer->slack; + + if (timer->slack < 0) /* auto slack: use 0.4% */ + expires_limit = expires + (expires - jiffies)/256; + + mask = expires ^ expires_limit; + + if (mask == 0) + return expires; + + bit = find_last_bit(&mask, BITS_PER_LONG); + + mask = (1 << bit) - 1; + + expires_limit = expires_limit & ~(mask); + + return expires_limit; +} + /** * mod_timer - modify a timer's timeout * @timer: the timer to be modified @@ -744,6 +798,8 @@ int mod_timer(struct timer_list *timer, unsigned long expires) if (timer_pending(timer) && timer->expires == expires) return 1; + expires = apply_slack(timer, expires); + return __mod_timer(timer, expires, false, TIMER_NOT_PINNED); } EXPORT_SYMBOL(mod_timer); -- cgit v1.2.3 From 351b3f7a21e413a9b14d0393171497d2373bd702 Mon Sep 17 00:00:00 2001 From: Carsten Emde Date: Fri, 2 Apr 2010 22:40:19 +0200 Subject: hrtimers: Provide schedule_hrtimeout for CLOCK_REALTIME The current version of schedule_hrtimeout() always uses the monotonic clock. Some system calls such as mq_timedsend() and mq_timedreceive(), however, require the use of the wall clock due to the definition of the system call. This patch provides the infrastructure to use schedule_hrtimeout() with a CLOCK_REALTIME timer. Signed-off-by: Carsten Emde Tested-by: Pradyumna Sampath Cc: Andrew Morton Cc: Arjan van de Veen LKML-Reference: <20100402204331.167439615@osadl.org> Signed-off-by: Thomas Gleixner --- include/linux/hrtimer.h | 2 ++ kernel/hrtimer.c | 67 ++++++++++++++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 5d86fb2309d..fd0c1b857d3 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -422,6 +422,8 @@ extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, extern int schedule_hrtimeout_range(ktime_t *expires, unsigned long delta, const enum hrtimer_mode mode); +extern int schedule_hrtimeout_range_clock(ktime_t *expires, + unsigned long delta, const enum hrtimer_mode mode, int clock); extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode); /* Soft interrupt function to run the hrtimer queues: */ diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 0086628b6e9..b9b134b3508 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -1749,35 +1749,15 @@ void __init hrtimers_init(void) } /** - * schedule_hrtimeout_range - sleep until timeout + * schedule_hrtimeout_range_clock - sleep until timeout * @expires: timeout value (ktime_t) * @delta: slack in expires timeout (ktime_t) * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL - * - * Make the current task sleep until the given expiry time has - * elapsed. The routine will return immediately unless - * the current task state has been set (see set_current_state()). - * - * The @delta argument gives the kernel the freedom to schedule the - * actual wakeup to a time that is both power and performance friendly. - * The kernel give the normal best effort behavior for "@expires+@delta", - * but may decide to fire the timer earlier, but no earlier than @expires. - * - * You can set the task state as follows - - * - * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to - * pass before the routine returns. - * - * %TASK_INTERRUPTIBLE - the routine may return early if a signal is - * delivered to the current task. - * - * The current task state is guaranteed to be TASK_RUNNING when this - * routine returns. - * - * Returns 0 when the timer has expired otherwise -EINTR + * @clock: timer clock, CLOCK_MONOTONIC or CLOCK_REALTIME */ -int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta, - const enum hrtimer_mode mode) +int __sched +schedule_hrtimeout_range_clock(ktime_t *expires, unsigned long delta, + const enum hrtimer_mode mode, int clock) { struct hrtimer_sleeper t; @@ -1799,7 +1779,7 @@ int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta, return -EINTR; } - hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, mode); + hrtimer_init_on_stack(&t.timer, clock, mode); hrtimer_set_expires_range_ns(&t.timer, *expires, delta); hrtimer_init_sleeper(&t, current); @@ -1818,6 +1798,41 @@ int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta, return !t.task ? 0 : -EINTR; } + +/** + * schedule_hrtimeout_range - sleep until timeout + * @expires: timeout value (ktime_t) + * @delta: slack in expires timeout (ktime_t) + * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL + * + * Make the current task sleep until the given expiry time has + * elapsed. The routine will return immediately unless + * the current task state has been set (see set_current_state()). + * + * The @delta argument gives the kernel the freedom to schedule the + * actual wakeup to a time that is both power and performance friendly. + * The kernel give the normal best effort behavior for "@expires+@delta", + * but may decide to fire the timer earlier, but no earlier than @expires. + * + * You can set the task state as follows - + * + * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to + * pass before the routine returns. + * + * %TASK_INTERRUPTIBLE - the routine may return early if a signal is + * delivered to the current task. + * + * The current task state is guaranteed to be TASK_RUNNING when this + * routine returns. + * + * Returns 0 when the timer has expired otherwise -EINTR + */ +int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta, + const enum hrtimer_mode mode) +{ + return schedule_hrtimeout_range_clock(expires, delta, mode, + CLOCK_MONOTONIC); +} EXPORT_SYMBOL_GPL(schedule_hrtimeout_range); /** -- cgit v1.2.3 From 9ca7d8e6834c40a99622bbe4a88aaf64313ae43c Mon Sep 17 00:00:00 2001 From: Carsten Emde Date: Fri, 2 Apr 2010 22:40:20 +0200 Subject: mqueue: Convert message queue timeout to use hrtimers The message queue functions mq_timedsend() and mq_timedreceive() have not yet been converted to use the hrtimer interface. This patch replaces the call to schedule_timeout() by a call to schedule_hrtimeout() and transforms the expiration time from timespec to ktime as required. [ tglx: Fixed whitespace wreckage ] Signed-off-by: Carsten Emde Tested-by: Pradyumna Sampath Cc: Arjan van de Veen Cc: Andrew Morton LKML-Reference: <20100402204331.715783034@osadl.org> Signed-off-by: Thomas Gleixner --- ipc/mqueue.c | 74 ++++++++++++++++++++---------------------------------------- 1 file changed, 25 insertions(+), 49 deletions(-) diff --git a/ipc/mqueue.c b/ipc/mqueue.c index e4e3f04803c..a9d8b0ced4e 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -428,7 +428,7 @@ static void wq_add(struct mqueue_inode_info *info, int sr, * sr: SEND or RECV */ static int wq_sleep(struct mqueue_inode_info *info, int sr, - long timeout, struct ext_wait_queue *ewp) + ktime_t *timeout, struct ext_wait_queue *ewp) { int retval; signed long time; @@ -439,7 +439,8 @@ static int wq_sleep(struct mqueue_inode_info *info, int sr, set_current_state(TASK_INTERRUPTIBLE); spin_unlock(&info->lock); - time = schedule_timeout(timeout); + time = schedule_hrtimeout_range_clock(timeout, + HRTIMER_MODE_ABS, 0, CLOCK_REALTIME); while (ewp->state == STATE_PENDING) cpu_relax(); @@ -551,31 +552,16 @@ static void __do_notify(struct mqueue_inode_info *info) wake_up(&info->wait_q); } -static long prepare_timeout(struct timespec *p) +static int prepare_timeout(const struct timespec __user *u_abs_timeout, + ktime_t *expires, struct timespec *ts) { - struct timespec nowts; - long timeout; - - if (p) { - if (unlikely(p->tv_nsec < 0 || p->tv_sec < 0 - || p->tv_nsec >= NSEC_PER_SEC)) - return -EINVAL; - nowts = CURRENT_TIME; - /* first subtract as jiffies can't be too big */ - p->tv_sec -= nowts.tv_sec; - if (p->tv_nsec < nowts.tv_nsec) { - p->tv_nsec += NSEC_PER_SEC; - p->tv_sec--; - } - p->tv_nsec -= nowts.tv_nsec; - if (p->tv_sec < 0) - return 0; - - timeout = timespec_to_jiffies(p) + 1; - } else - return MAX_SCHEDULE_TIMEOUT; + if (copy_from_user(ts, u_abs_timeout, sizeof(struct timespec))) + return -EFAULT; + if (!timespec_valid(ts)) + return -EINVAL; - return timeout; + *expires = timespec_to_ktime(*ts); + return 0; } static void remove_notification(struct mqueue_inode_info *info) @@ -861,22 +847,21 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr, struct ext_wait_queue *receiver; struct msg_msg *msg_ptr; struct mqueue_inode_info *info; - struct timespec ts, *p = NULL; - long timeout; + ktime_t expires, *timeout = NULL; + struct timespec ts; int ret; if (u_abs_timeout) { - if (copy_from_user(&ts, u_abs_timeout, - sizeof(struct timespec))) - return -EFAULT; - p = &ts; + int res = prepare_timeout(u_abs_timeout, &expires, &ts); + if (res) + return res; + timeout = &expires; } if (unlikely(msg_prio >= (unsigned long) MQ_PRIO_MAX)) return -EINVAL; - audit_mq_sendrecv(mqdes, msg_len, msg_prio, p); - timeout = prepare_timeout(p); + audit_mq_sendrecv(mqdes, msg_len, msg_prio, timeout ? &ts : NULL); filp = fget(mqdes); if (unlikely(!filp)) { @@ -918,9 +903,6 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr, if (filp->f_flags & O_NONBLOCK) { spin_unlock(&info->lock); ret = -EAGAIN; - } else if (unlikely(timeout < 0)) { - spin_unlock(&info->lock); - ret = timeout; } else { wait.task = current; wait.msg = (void *) msg_ptr; @@ -953,24 +935,23 @@ SYSCALL_DEFINE5(mq_timedreceive, mqd_t, mqdes, char __user *, u_msg_ptr, size_t, msg_len, unsigned int __user *, u_msg_prio, const struct timespec __user *, u_abs_timeout) { - long timeout; ssize_t ret; struct msg_msg *msg_ptr; struct file *filp; struct inode *inode; struct mqueue_inode_info *info; struct ext_wait_queue wait; - struct timespec ts, *p = NULL; + ktime_t expires, *timeout = NULL; + struct timespec ts; if (u_abs_timeout) { - if (copy_from_user(&ts, u_abs_timeout, - sizeof(struct timespec))) - return -EFAULT; - p = &ts; + int res = prepare_timeout(u_abs_timeout, &expires, &ts); + if (res) + return res; + timeout = &expires; } - audit_mq_sendrecv(mqdes, msg_len, 0, p); - timeout = prepare_timeout(p); + audit_mq_sendrecv(mqdes, msg_len, 0, timeout ? &ts : NULL); filp = fget(mqdes); if (unlikely(!filp)) { @@ -1002,11 +983,6 @@ SYSCALL_DEFINE5(mq_timedreceive, mqd_t, mqdes, char __user *, u_msg_ptr, if (filp->f_flags & O_NONBLOCK) { spin_unlock(&info->lock); ret = -EAGAIN; - msg_ptr = NULL; - } else if (unlikely(timeout < 0)) { - spin_unlock(&info->lock); - ret = timeout; - msg_ptr = NULL; } else { wait.task = current; wait.state = STATE_NONE; -- cgit v1.2.3 From 6a867a395558a7f882d041783e4cdea6744ca2bf Mon Sep 17 00:00:00 2001 From: John Stultz Date: Tue, 6 Apr 2010 14:30:51 -0700 Subject: time: Remove xtime_cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the earlier logarithmic time accumulation patch, xtime will now always be within one "tick" of the current time, instead of possibly half a second off. This removes the need for the xtime_cache value, which always stored the time at the last interrupt, so this patch cleans that up removing the xtime_cache related code. This patch also addresses an issue with an earlier version of this change, where xtime_cache was normalizing xtime, which could in some cases be not valid (ie: tv_nsec == NSEC_PER_SEC). This is fixed by handling the edge case in update_wall_time(). Signed-off-by: John Stultz Cc: Petr Titěra LKML-Reference: <1270589451-30773-1-git-send-email-johnstul@us.ibm.com> Signed-off-by: Thomas Gleixner --- include/linux/time.h | 1 - kernel/time/timekeeping.c | 35 ++++++++++++++++------------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/include/linux/time.h b/include/linux/time.h index 6e026e45a17..ea3559f0b3f 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -150,7 +150,6 @@ extern struct timespec timespec_trunc(struct timespec t, unsigned gran); extern int timekeeping_valid_for_hres(void); extern u64 timekeeping_max_deferment(void); extern void update_wall_time(void); -extern void update_xtime_cache(u64 nsec); extern void timekeeping_leap_insert(int leapsecond); struct tms; diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 16736379a9c..1137f245a4b 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -165,13 +165,6 @@ struct timespec raw_time; /* flag for if timekeeping is suspended */ int __read_mostly timekeeping_suspended; -static struct timespec xtime_cache __attribute__ ((aligned (16))); -void update_xtime_cache(u64 nsec) -{ - xtime_cache = xtime; - timespec_add_ns(&xtime_cache, nsec); -} - /* must hold xtime_lock */ void timekeeping_leap_insert(int leapsecond) { @@ -332,8 +325,6 @@ int do_settimeofday(struct timespec *tv) xtime = *tv; - update_xtime_cache(0); - timekeeper.ntp_error = 0; ntp_clear(); @@ -559,7 +550,6 @@ void __init timekeeping_init(void) } set_normalized_timespec(&wall_to_monotonic, -boot.tv_sec, -boot.tv_nsec); - update_xtime_cache(0); total_sleep_time.tv_sec = 0; total_sleep_time.tv_nsec = 0; write_sequnlock_irqrestore(&xtime_lock, flags); @@ -593,7 +583,6 @@ static int timekeeping_resume(struct sys_device *dev) wall_to_monotonic = timespec_sub(wall_to_monotonic, ts); total_sleep_time = timespec_add_safe(total_sleep_time, ts); } - update_xtime_cache(0); /* re-base the last cycle value */ timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); timekeeper.ntp_error = 0; @@ -788,7 +777,6 @@ void update_wall_time(void) { struct clocksource *clock; cycle_t offset; - u64 nsecs; int shift = 0, maxshift; /* Make sure we're fully resumed: */ @@ -846,7 +834,9 @@ void update_wall_time(void) timekeeper.ntp_error += neg << timekeeper.ntp_error_shift; } - /* store full nanoseconds into xtime after rounding it up and + + /* + * Store full nanoseconds into xtime after rounding it up and * add the remainder to the error difference. */ xtime.tv_nsec = ((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1; @@ -854,8 +844,15 @@ void update_wall_time(void) timekeeper.ntp_error += timekeeper.xtime_nsec << timekeeper.ntp_error_shift; - nsecs = clocksource_cyc2ns(offset, timekeeper.mult, timekeeper.shift); - update_xtime_cache(nsecs); + /* + * Finally, make sure that after the rounding + * xtime.tv_nsec isn't larger then NSEC_PER_SEC + */ + if (unlikely(xtime.tv_nsec >= NSEC_PER_SEC)) { + xtime.tv_nsec -= NSEC_PER_SEC; + xtime.tv_sec++; + second_overflow(); + } /* check to see if there is a new clocksource to use */ update_vsyscall(&xtime, timekeeper.clock, timekeeper.mult); @@ -895,13 +892,13 @@ EXPORT_SYMBOL_GPL(monotonic_to_bootbased); unsigned long get_seconds(void) { - return xtime_cache.tv_sec; + return xtime.tv_sec; } EXPORT_SYMBOL(get_seconds); struct timespec __current_kernel_time(void) { - return xtime_cache; + return xtime; } struct timespec current_kernel_time(void) @@ -912,7 +909,7 @@ struct timespec current_kernel_time(void) do { seq = read_seqbegin(&xtime_lock); - now = xtime_cache; + now = xtime; } while (read_seqretry(&xtime_lock, seq)); return now; @@ -927,7 +924,7 @@ struct timespec get_monotonic_coarse(void) do { seq = read_seqbegin(&xtime_lock); - now = xtime_cache; + now = xtime; mono = wall_to_monotonic; } while (read_seqretry(&xtime_lock, seq)); -- cgit v1.2.3 From 29f87b793da421a6ab816d991dc8dbf909dfb66a Mon Sep 17 00:00:00 2001 From: Stanislaw Gruszka Date: Tue, 27 Apr 2010 14:12:15 -0700 Subject: posix-cpu-timers: Optimize run_posix_cpu_timers() We can optimize and simplify things taking into account signal->cputimer is always running when we have configured any process wide cpu timer. In check_process_timers(), we don't have to check if new updated value of signal->cputime_expires is smaller, since we maintain new first expiration time ({prof,virt,sched}_expires) in code flow and all other writes to expiration cache are protected by sighand->siglock . Signed-off-by: Stanislaw Gruszka Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Hidetoshi Seto Cc: Balbir Singh Signed-off-by: Andrew Morton Signed-off-by: Thomas Gleixner --- kernel/posix-cpu-timers.c | 80 +++++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 51 deletions(-) diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 799f360d147..00bb252f29a 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c @@ -1002,16 +1002,9 @@ static void stop_process_timers(struct signal_struct *sig) struct thread_group_cputimer *cputimer = &sig->cputimer; unsigned long flags; - if (!cputimer->running) - return; - spin_lock_irqsave(&cputimer->lock, flags); cputimer->running = 0; spin_unlock_irqrestore(&cputimer->lock, flags); - - sig->cputime_expires.prof_exp = cputime_zero; - sig->cputime_expires.virt_exp = cputime_zero; - sig->cputime_expires.sched_exp = 0; } static u32 onecputick; @@ -1048,6 +1041,23 @@ static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, } } +/** + * task_cputime_zero - Check a task_cputime struct for all zero fields. + * + * @cputime: The struct to compare. + * + * Checks @cputime to see if all fields are zero. Returns true if all fields + * are zero, false if any field is nonzero. + */ +static inline int task_cputime_zero(const struct task_cputime *cputime) +{ + if (cputime_eq(cputime->utime, cputime_zero) && + cputime_eq(cputime->stime, cputime_zero) && + cputime->sum_exec_runtime == 0) + return 1; + return 0; +} + /* * Check for any per-thread CPU timers that have fired and move them * off the tsk->*_timers list onto the firing list. Per-thread timers @@ -1064,19 +1074,6 @@ static void check_process_timers(struct task_struct *tsk, struct task_cputime cputime; unsigned long soft; - /* - * Don't sample the current process CPU clocks if there are no timers. - */ - if (list_empty(&timers[CPUCLOCK_PROF]) && - cputime_eq(sig->it[CPUCLOCK_PROF].expires, cputime_zero) && - sig->rlim[RLIMIT_CPU].rlim_cur == RLIM_INFINITY && - list_empty(&timers[CPUCLOCK_VIRT]) && - cputime_eq(sig->it[CPUCLOCK_VIRT].expires, cputime_zero) && - list_empty(&timers[CPUCLOCK_SCHED])) { - stop_process_timers(sig); - return; - } - /* * Collect the current process totals. */ @@ -1166,18 +1163,11 @@ static void check_process_timers(struct task_struct *tsk, } } - if (!cputime_eq(prof_expires, cputime_zero) && - (cputime_eq(sig->cputime_expires.prof_exp, cputime_zero) || - cputime_gt(sig->cputime_expires.prof_exp, prof_expires))) - sig->cputime_expires.prof_exp = prof_expires; - if (!cputime_eq(virt_expires, cputime_zero) && - (cputime_eq(sig->cputime_expires.virt_exp, cputime_zero) || - cputime_gt(sig->cputime_expires.virt_exp, virt_expires))) - sig->cputime_expires.virt_exp = virt_expires; - if (sched_expires != 0 && - (sig->cputime_expires.sched_exp == 0 || - sig->cputime_expires.sched_exp > sched_expires)) - sig->cputime_expires.sched_exp = sched_expires; + sig->cputime_expires.prof_exp = prof_expires; + sig->cputime_expires.virt_exp = virt_expires; + sig->cputime_expires.sched_exp = sched_expires; + if (task_cputime_zero(&sig->cputime_expires)) + stop_process_timers(sig); } /* @@ -1249,23 +1239,6 @@ out: ++timer->it_requeue_pending; } -/** - * task_cputime_zero - Check a task_cputime struct for all zero fields. - * - * @cputime: The struct to compare. - * - * Checks @cputime to see if all fields are zero. Returns true if all fields - * are zero, false if any field is nonzero. - */ -static inline int task_cputime_zero(const struct task_cputime *cputime) -{ - if (cputime_eq(cputime->utime, cputime_zero) && - cputime_eq(cputime->stime, cputime_zero) && - cputime->sum_exec_runtime == 0) - return 1; - return 0; -} - /** * task_cputime_expired - Compare two task_cputime entities. * @@ -1322,7 +1295,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk) } sig = tsk->signal; - if (!task_cputime_zero(&sig->cputime_expires)) { + if (sig->cputimer.running) { struct task_cputime group_sample; thread_group_cputimer(tsk, &group_sample); @@ -1359,7 +1332,12 @@ void run_posix_cpu_timers(struct task_struct *tsk) * put them on the firing list. */ check_thread_timers(tsk, &firing); - check_process_timers(tsk, &firing); + /* + * If there are any active process wide timers (POSIX 1.b, itimers, + * RLIMIT_CPU) cputimer must be running. + */ + if (tsk->signal->cputimer.running) + check_process_timers(tsk, &firing); /* * We must release these locks before taking any timer's lock. -- cgit v1.2.3 From d7e81c269db899b800e0963dc4aceece1f82a680 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 7 May 2010 18:07:38 -0700 Subject: clocksource: Add clocksource_register_hz/khz interface How to pick good mult/shift pairs has always been difficult to describe to folks writing clocksource drivers, since it requires careful tradeoffs in adjustment accuracy vs overflow limits. Now, with the clocks_calc_mult_shift function, its much easier. However, not many clocksources have converted to using that function, and there is still the issue of the max interval length assumption being made by each clocksource driver independently. So this patch simplifies the registration process by having clocksources be registered with a hz/khz value and the registration function taking care of setting mult/shift. This should take most of the confusion out of writing a clocksource driver. Additionally it also keeps the shift size tradeoff (more accuracy vs longer possible nohz times) centralized so the timekeeping core can keep track of the assumptions being made. [ tglx: Coding style and comments fixed ] Signed-off-by: John Stultz LKML-Reference: <1273280858-30143-1-git-send-email-johnstul@us.ibm.com> Signed-off-by: Thomas Gleixner --- include/linux/clocksource.h | 19 +++++++++++++++++- kernel/time/clocksource.c | 48 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index 4bca8b60cdf..5ea3c60c160 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -273,7 +273,6 @@ static inline s64 clocksource_cyc2ns(cycle_t cycles, u32 mult, u32 shift) } -/* used to install a new clocksource */ extern int clocksource_register(struct clocksource*); extern void clocksource_unregister(struct clocksource*); extern void clocksource_touch_watchdog(void); @@ -287,6 +286,24 @@ extern void clocksource_mark_unstable(struct clocksource *cs); extern void clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 minsec); +/* + * Don't call __clocksource_register_scale directly, use + * clocksource_register_hz/khz + */ +extern int +__clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq); + +static inline int clocksource_register_hz(struct clocksource *cs, u32 hz) +{ + return __clocksource_register_scale(cs, 1, hz); +} + +static inline int clocksource_register_khz(struct clocksource *cs, u32 khz) +{ + return __clocksource_register_scale(cs, 1000, khz); +} + + static inline void clocksource_calc_mult_shift(struct clocksource *cs, u32 freq, u32 minsec) { diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 1f5dde63745..f08e99c1d56 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -625,6 +625,54 @@ static void clocksource_enqueue(struct clocksource *cs) list_add(&cs->list, entry); } + +/* + * Maximum time we expect to go between ticks. This includes idle + * tickless time. It provides the trade off between selecting a + * mult/shift pair that is very precise but can only handle a short + * period of time, vs. a mult/shift pair that can handle long periods + * of time but isn't as precise. + * + * This is a subsystem constant, and actual hardware limitations + * may override it (ie: clocksources that wrap every 3 seconds). + */ +#define MAX_UPDATE_LENGTH 5 /* Seconds */ + +/** + * __clocksource_register_scale - Used to install new clocksources + * @t: clocksource to be registered + * @scale: Scale factor multiplied against freq to get clocksource hz + * @freq: clocksource frequency (cycles per second) divided by scale + * + * Returns -EBUSY if registration fails, zero otherwise. + * + * This *SHOULD NOT* be called directly! Please use the + * clocksource_register_hz() or clocksource_register_khz helper functions. + */ +int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq) +{ + + /* + * Ideally we want to use some of the limits used in + * clocksource_max_deferment, to provide a more informed + * MAX_UPDATE_LENGTH. But for now this just gets the + * register interface working properly. + */ + clocks_calc_mult_shift(&cs->mult, &cs->shift, freq, + NSEC_PER_SEC/scale, + MAX_UPDATE_LENGTH*scale); + cs->max_idle_ns = clocksource_max_deferment(cs); + + mutex_lock(&clocksource_mutex); + clocksource_enqueue(cs); + clocksource_select(); + clocksource_enqueue_watchdog(cs); + mutex_unlock(&clocksource_mutex); + return 0; +} +EXPORT_SYMBOL_GPL(__clocksource_register_scale); + + /** * clocksource_register - Used to install new clocksources * @t: clocksource to be registered -- cgit v1.2.3