From cd1334f03f7b799bc6893b511daf2080e8f73863 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:19 -0700 Subject: gru: bug fixes for GRU exception handling Bug fixes for GRU exception handling. Additional fields from the CBR must be returned to the user to allow the user to correctly diagnose GRU exceptions. Handle endcase in TFH TLB miss handling. Verify that TFH actually indicates a pending exception. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index ab118558552e..4089f862aa29 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -334,6 +334,8 @@ static int gru_try_dropin(struct gru_thread_state *gts, * Might be a hardware race OR a stupid user. Ignore FMM because FMM * is a transient state. */ + if (tfh->status != TFHSTATUS_EXCEPTION) + goto failnoexception; if (tfh->state == TFHSTATE_IDLE) goto failidle; if (tfh->state == TFHSTATE_MISS_FMM && cb) @@ -401,8 +403,17 @@ failfmm: gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state); return 0; +failnoexception: + /* TFH status did not show exception pending */ + gru_flush_cache(tfh); + if (cb) + gru_flush_cache(cb); + STAT(tlb_dropin_fail_no_exception); + gru_dbg(grudev, "FAILED non-exception tfh: 0x%p, status %d, state %d\n", tfh, tfh->status, tfh->state); + return 0; + failidle: - /* TFH was idle - no miss pending */ + /* TFH state was idle - no miss pending */ gru_flush_cache(tfh); if (cb) gru_flush_cache(cb); @@ -472,7 +483,8 @@ irqreturn_t gru_intr(int irq, void *dev_id) * This is running in interrupt context. Trylock the mmap_sem. * If it fails, retry the fault in user context. */ - if (down_read_trylock(>s->ts_mm->mmap_sem)) { + if (!gts->ts_force_cch_reload && + down_read_trylock(>s->ts_mm->mmap_sem)) { gru_try_dropin(gts, tfh, NULL); up_read(>s->ts_mm->mmap_sem); } else { @@ -595,14 +607,19 @@ int gru_get_exception_detail(unsigned long arg) excdet.ecause = cbe->ecause; excdet.exceptdet0 = cbe->idef1upd; excdet.exceptdet1 = cbe->idef3upd; + excdet.cbrstate = cbe->cbrstate; + excdet.cbrexecstatus = cbe->cbrexecstatus; ret = 0; } else { ret = -EAGAIN; } gru_unlock_gts(gts); - gru_dbg(grudev, "address 0x%lx, ecause 0x%x\n", excdet.cb, - excdet.ecause); + gru_dbg(grudev, + "cb 0x%lx, op %d, exopc %d, cbrstate %d, cbrexecstatus 0x%x, ecause 0x%x, " + "exdet0 0x%lx, exdet1 0x%x\n", + excdet.cb, excdet.opc, excdet.exopc, excdet.cbrstate, excdet.cbrexecstatus, + excdet.ecause, excdet.exceptdet0, excdet.exceptdet1); if (!ret && copy_to_user((void __user *)arg, &excdet, sizeof(excdet))) ret = -EFAULT; return ret; -- cgit v1.2.3 From d57c82b10709bbb1deb7eb26cf42abcde8851e4d Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:20 -0700 Subject: gru: change context load and unload Remove "static" from the functions for loading/unloading GRU contexts. These functions will be called from other GRU files. Fix bug in unlocking gru context. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 4089f862aa29..f15152165a99 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -558,8 +558,8 @@ int gru_handle_user_call_os(unsigned long cb) * CCH may contain stale data if ts_force_cch_reload is set. */ if (gts->ts_gru && gts->ts_force_cch_reload) { - gru_update_cch(gts, 0); gts->ts_force_cch_reload = 0; + gru_update_cch(gts, 0); } ret = -EAGAIN; @@ -644,7 +644,7 @@ static int gru_unload_all_contexts(void) if (gts && mutex_trylock(>s->ts_ctxlock)) { spin_unlock(&gru->gs_lock); gru_unload_context(gts, 1); - gru_unlock_gts(gts); + mutex_unlock(>s->ts_ctxlock); spin_lock(&gru->gs_lock); } } -- cgit v1.2.3 From 4a7a17c1188a878e9f00e4ca8dc724c7cff17606 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:25 -0700 Subject: gru: support instruction completion interrupts Add support for interrupts generated by GRU instruction completion. Previously, the only interrupts were for TLB misses. The hardware also supports interrupts on instruction completion. This will be supported for instructions issued by the kernel. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index f15152165a99..3220e95be6b5 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -166,7 +166,8 @@ static inline struct gru_state *irq_to_gru(int irq) * the GRU, atomic operations must be used to clear bits. */ static void get_clear_fault_map(struct gru_state *gru, - struct gru_tlb_fault_map *map) + struct gru_tlb_fault_map *imap, + struct gru_tlb_fault_map *dmap) { unsigned long i, k; struct gru_tlb_fault_map *tfm; @@ -177,7 +178,11 @@ static void get_clear_fault_map(struct gru_state *gru, k = tfm->fault_bits[i]; if (k) k = xchg(&tfm->fault_bits[i], 0UL); - map->fault_bits[i] = k; + imap->fault_bits[i] = k; + k = tfm->done_bits[i]; + if (k) + k = xchg(&tfm->done_bits[i], 0UL); + dmap->fault_bits[i] = k; } /* @@ -449,7 +454,7 @@ failactive: irqreturn_t gru_intr(int irq, void *dev_id) { struct gru_state *gru; - struct gru_tlb_fault_map map; + struct gru_tlb_fault_map imap, dmap; struct gru_thread_state *gts; struct gru_tlb_fault_handle *tfh = NULL; int cbrnum, ctxnum; @@ -462,11 +467,19 @@ irqreturn_t gru_intr(int irq, void *dev_id) raw_smp_processor_id(), irq); return IRQ_NONE; } - get_clear_fault_map(gru, &map); - gru_dbg(grudev, "irq %d, gru %x, map 0x%lx\n", irq, gru->gs_gid, - map.fault_bits[0]); + get_clear_fault_map(gru, &imap, &dmap); + gru_dbg(grudev, + "irq %d, gid %d, imap %016lx %016lx, dmap %016lx %016lx\n", + irq, gru->gs_gid, dmap.fault_bits[0], dmap.fault_bits[1], + dmap.fault_bits[0], dmap.fault_bits[1]); + + for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) { + complete(gru->gs_blade->bs_async_wq); + gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n", + gru->gs_gid, cbrnum, gru->gs_blade->bs_async_wq->done); + } - for_each_cbr_in_tfm(cbrnum, map.fault_bits) { + for_each_cbr_in_tfm(cbrnum, imap.fault_bits) { tfh = get_tfh_by_index(gru, cbrnum); prefetchw(tfh); /* Helps on hdw, required for emulator */ -- cgit v1.2.3 From 9120dec47f150636d85b3dba03318ccecd181c79 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:25 -0700 Subject: gru: support for asynchronous gru instructions Add support for asynchronous GRU instructions. Currently, asynchronous instructions are supported only for GRU instructions issued by the kernel. [akpm@linux-foundation.org: build fix] Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 3220e95be6b5..8443e90f9f6c 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -468,10 +468,6 @@ irqreturn_t gru_intr(int irq, void *dev_id) return IRQ_NONE; } get_clear_fault_map(gru, &imap, &dmap); - gru_dbg(grudev, - "irq %d, gid %d, imap %016lx %016lx, dmap %016lx %016lx\n", - irq, gru->gs_gid, dmap.fault_bits[0], dmap.fault_bits[1], - dmap.fault_bits[0], dmap.fault_bits[1]); for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) { complete(gru->gs_blade->bs_async_wq); -- cgit v1.2.3 From 270952a907220c0331fdaecbb55df892921c5e2d Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:27 -0700 Subject: gru: update to rev 0.9 of gru spec Update GRU driver to the latest version of the GRU spec. This consists of minor updates: - changes & additions to error status bits - new restriction on handling of TLB misses while in FMM mode - new field (not used by software) in TFH Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 8443e90f9f6c..a489807613f8 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -339,8 +339,12 @@ static int gru_try_dropin(struct gru_thread_state *gts, * Might be a hardware race OR a stupid user. Ignore FMM because FMM * is a transient state. */ - if (tfh->status != TFHSTATUS_EXCEPTION) - goto failnoexception; + if (tfh->status != TFHSTATUS_EXCEPTION) { + gru_flush_cache(tfh); + if (tfh->status != TFHSTATUS_EXCEPTION) + goto failnoexception; + STAT(tfh_stale_on_fault); + } if (tfh->state == TFHSTATE_IDLE) goto failidle; if (tfh->state == TFHSTATE_MISS_FMM && cb) -- cgit v1.2.3 From 1a2c09e3b41e334b6651d53b39cfe8ceefbc45f8 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:28 -0700 Subject: gru: fix cache coherency issues with instruction retry Fix two problems related to GRU instruction failures. Cache coherency is not maintained for CBEs except when loading or unloading contexts. When reading a CBE to extract error information, the CBE must first be flushed from the cache. The function that reads kerrnel CBEs was reading the wrong CBE. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index a489807613f8..6d0681236db5 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -614,7 +614,7 @@ int gru_get_exception_detail(unsigned long arg) } else if (gts->ts_gru) { cbrnum = thread_cbr_number(gts, ucbnum); cbe = get_cbe_by_index(gts->ts_gru, cbrnum); - prefetchw(cbe);/* Harmless on hardware, required for emulator */ + gru_flush_cache(cbe); /* CBE not coherent */ excdet.opc = cbe->opccpy; excdet.exopc = cbe->exopccpy; excdet.ecause = cbe->ecause; @@ -622,6 +622,7 @@ int gru_get_exception_detail(unsigned long arg) excdet.exceptdet1 = cbe->idef3upd; excdet.cbrstate = cbe->cbrstate; excdet.cbrexecstatus = cbe->cbrexecstatus; + gru_flush_cache(cbe); ret = 0; } else { ret = -EAGAIN; -- cgit v1.2.3 From 7e796a72a2691d7094fd62da61097294d0d59ce4 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:30 -0700 Subject: gru: collect per-context user statistics Collect GRU statistics for each user GRU context. Statistics are kept for TLB misses & content resource contention. Add user request for retrieving the statistics. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 6d0681236db5..cdd151b30dc7 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -498,6 +498,7 @@ irqreturn_t gru_intr(int irq, void *dev_id) */ if (!gts->ts_force_cch_reload && down_read_trylock(>s->ts_mm->mmap_sem)) { + gts->ustats.fmm_tlbdropin++; gru_try_dropin(gts, tfh, NULL); up_read(>s->ts_mm->mmap_sem); } else { @@ -516,6 +517,7 @@ static int gru_user_dropin(struct gru_thread_state *gts, struct gru_mm_struct *gms = gts->ts_gms; int ret; + gts->ustats.upm_tlbdropin++; while (1) { wait_event(gms->ms_wait_queue, atomic_read(&gms->ms_range_active) == 0); @@ -718,6 +720,31 @@ int gru_user_flush_tlb(unsigned long arg) return 0; } +/* + * Fetch GSEG statisticss + */ +long gru_get_gseg_statistics(unsigned long arg) +{ + struct gru_thread_state *gts; + struct gru_get_gseg_statistics_req req; + + if (copy_from_user(&req, (void __user *)arg, sizeof(req))) + return -EFAULT; + + gts = gru_find_lock_gts(req.gseg); + if (gts) { + memcpy(&req.stats, >s->ustats, sizeof(gts->ustats)); + gru_unlock_gts(gts); + } else { + memset(&req.stats, 0, sizeof(gts->ustats)); + } + + if (copy_to_user((void __user *)arg, &req, sizeof(req))) + return -EFAULT; + + return 0; +} + /* * Register the current task as the user of the GSEG slice. * Needed for TLB fault interrupt targeting. -- cgit v1.2.3 From 92b39388eeb45326feb0fa8bd69dbbce66c9efbf Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:32 -0700 Subject: gru: generic infrastructure for context options Change the user GRU request for specifying the "task_slice" option to use a generic infrastructure that can be expanded in the future to include additional context options. No new capabilities are added with this patch. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index cdd151b30dc7..b894b7ed9c35 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -749,18 +749,30 @@ long gru_get_gseg_statistics(unsigned long arg) * Register the current task as the user of the GSEG slice. * Needed for TLB fault interrupt targeting. */ -int gru_set_task_slice(long address) +int gru_set_context_option(unsigned long arg) { struct gru_thread_state *gts; + struct gru_set_context_option_req req; + int ret = 0; - STAT(set_task_slice); - gru_dbg(grudev, "address 0x%lx\n", address); - gts = gru_alloc_locked_gts(address); + STAT(set_context_option); + if (copy_from_user(&req, (void __user *)arg, sizeof(req))) + return -EFAULT; + gru_dbg(grudev, "op %d, gseg 0x%lx, value1 0x%lx\n", req.op, req.gseg, req.val1); + + gts = gru_alloc_locked_gts(req.gseg); if (!gts) return -EINVAL; - gts->ts_tgid_owner = current->tgid; + switch (req.op) { + case sco_gseg_owner: + /* Register the current task as the GSEG owner */ + gts->ts_tgid_owner = current->tgid; + break; + default: + ret = -EINVAL; + } gru_unlock_gts(gts); - return 0; + return ret; } -- cgit v1.2.3 From b1b19fcfa417cf62447413d6e8b9b6598adf00b9 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:33 -0700 Subject: gru: add user request to specify gru slice Add a user request to specify the gru instruction slice parameter for user contexts. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index b894b7ed9c35..1ad360cd3183 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -769,6 +769,10 @@ int gru_set_context_option(unsigned long arg) /* Register the current task as the GSEG owner */ gts->ts_tgid_owner = current->tgid; break; + case sco_cch_req_slice: + /* Set the CCH slice option */ + gts->ts_cch_req_slice = req.val1 & 3; + break; default: ret = -EINVAL; } -- cgit v1.2.3 From 1926ee85a903d189c5702eed6531be321e33eb47 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 17 Jun 2009 16:28:33 -0700 Subject: gru: fix potential use-after-free when purging GRU tlbs Fix potential SGI GRU bug that could cause a use-after-free. If one thread in a task is flushing the GRU and another thread destroys the GRU context, there is the potential to access a table after it has been freed. Copy the gms pointer to a local variable before unlocking the gts table. Note that no refcnt is needed for the gms - the reference is held indirectly by the task's mm_struct. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers/misc/sgi-gru/grufault.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 1ad360cd3183..679e01778286 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -702,6 +702,7 @@ int gru_user_flush_tlb(unsigned long arg) { struct gru_thread_state *gts; struct gru_flush_tlb_req req; + struct gru_mm_struct *gms; STAT(user_flush_tlb); if (copy_from_user(&req, (void __user *)arg, sizeof(req))) @@ -714,8 +715,9 @@ int gru_user_flush_tlb(unsigned long arg) if (!gts) return -EINVAL; - gru_flush_tlb_range(gts->ts_gms, req.vaddr, req.len); + gms = gts->ts_gms; gru_unlock_gts(gts); + gru_flush_tlb_range(gms, req.vaddr, req.len); return 0; } -- cgit v1.2.3