summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPeter Zijlstra <peterz@infradead.org>2011-07-28 10:43:51 +0200
committerClark Williams <williams@redhat.com>2012-04-04 09:14:57 -0500
commitc2169355ac9f6e0052e0a1a94d33ff575ab9b05b (patch)
treee8b2f6e0c51c4ede77ff4b62db2848e403b5bb52
parentdafda62b8cdb123ca4c1d066833437f3104b66a8 (diff)
mm, rt: kmap_atomic scheduling
In fact, with migrate_disable() existing one could play games with kmap_atomic. You could save/restore the kmap_atomic slots on context switch (if there are any in use of course), this should be esp easy now that we have a kmap_atomic stack. Something like the below.. it wants replacing all the preempt_disable() stuff with pagefault_disable() && migrate_disable() of course, but then you can flip kmaps around like below. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> [dvhart@linux.intel.com: build fix] Link: http://lkml.kernel.org/r/1311842631.5890.208.camel@twins
-rw-r--r--arch/x86/kernel/process_32.c36
-rw-r--r--include/linux/sched.h5
-rw-r--r--mm/memory.c2
3 files changed, 43 insertions, 0 deletions
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index ada175e33206..20f15731ec90 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -39,6 +39,7 @@
#include <linux/io.h>
#include <linux/kdebug.h>
#include <linux/cpuidle.h>
+#include <linux/highmem.h>
#include <asm/pgtable.h>
#include <asm/system.h>
@@ -339,6 +340,41 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
__switch_to_xtra(prev_p, next_p, tss);
+#if defined CONFIG_PREEMPT_RT_FULL && defined CONFIG_HIGHMEM
+ /*
+ * Save @prev's kmap_atomic stack
+ */
+ prev_p->kmap_idx = __this_cpu_read(__kmap_atomic_idx);
+ if (unlikely(prev_p->kmap_idx)) {
+ int i;
+
+ for (i = 0; i < prev_p->kmap_idx; i++) {
+ int idx = i + KM_TYPE_NR * smp_processor_id();
+
+ pte_t *ptep = kmap_pte - idx;
+ prev_p->kmap_pte[i] = *ptep;
+ kpte_clear_flush(ptep, __fix_to_virt(FIX_KMAP_BEGIN + idx));
+ }
+
+ __this_cpu_write(__kmap_atomic_idx, 0);
+ }
+
+ /*
+ * Restore @next_p's kmap_atomic stack
+ */
+ if (unlikely(next_p->kmap_idx)) {
+ int i;
+
+ __this_cpu_write(__kmap_atomic_idx, next_p->kmap_idx);
+
+ for (i = 0; i < next_p->kmap_idx; i++) {
+ int idx = i + KM_TYPE_NR * smp_processor_id();
+
+ set_pte(kmap_pte - idx, next_p->kmap_pte[i]);
+ }
+ }
+#endif
+
/*
* Leave lazy mode, flushing any hypercalls made here.
* This must be done before restoring TLS segments so
diff --git a/include/linux/sched.h b/include/linux/sched.h
index a84a90152121..1f6b11ad5312 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -63,6 +63,7 @@ struct sched_param {
#include <linux/nodemask.h>
#include <linux/mm_types.h>
+#include <asm/kmap_types.h>
#include <asm/system.h>
#include <asm/page.h>
#include <asm/ptrace.h>
@@ -1603,6 +1604,10 @@ struct task_struct {
struct rcu_head put_rcu;
int softirq_nestcnt;
#endif
+#if defined CONFIG_PREEMPT_RT_FULL && defined CONFIG_HIGHMEM
+ int kmap_idx;
+ pte_t kmap_pte[KM_TYPE_NR];
+#endif
};
#ifdef CONFIG_PREEMPT_RT_FULL
diff --git a/mm/memory.c b/mm/memory.c
index 0d248c51a411..61744e9432cf 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3439,6 +3439,7 @@ unlock:
#ifdef CONFIG_PREEMPT_RT_FULL
void pagefault_disable(void)
{
+ migrate_disable();
current->pagefault_disabled++;
/*
* make sure to have issued the store before a pagefault
@@ -3456,6 +3457,7 @@ void pagefault_enable(void)
*/
barrier();
current->pagefault_disabled--;
+ migrate_enable();
}
EXPORT_SYMBOL_GPL(pagefault_enable);
#endif