scheduler: Re-compute time-average nr_running on read

Port commit 1802afb2a (http://git-master/r/111637) from v3.1

Re-compute time-average nr_running when it is read. This would
prevent reading stalled average value if there were no run-queue
changes for a long time. New average value is returned to the reader,
but not stored to avoid concurrent writes. Light-weight sequential
counter synchronization is used to assure data consistency for
re-computing average.

Original author: Alex Frid <afrid@nvidia.com>

Signed-off-by: Alex Frid <afrid@nvidia.com>
Signed-off-by: Varun Wadekar <vwadekar@nvidia.com>

Change-Id: Ic486006d62436fb61cda4ab6897e933f5c102b52

Rebase-Id: Re0ca57c84a644e8e2d474930379cdcc386a2135a

2 files changed, 42 insertions, 17 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 014040fa3d21..4bde4c7617eb 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2075,6 +2075,32 @@ unsigned long this_cpu_load(void)
 	return this->cpu_load[0];
 }
 
+unsigned long avg_nr_running(void)
+{
+	unsigned long i, sum = 0;
+	unsigned int seqcnt, ave_nr_running;
+
+	for_each_online_cpu(i) {
+		struct rq *q = cpu_rq(i);
+
+		/*
+		 * Update average to avoid reading stalled value if there were
+		 * no run-queue changes for a long time. On the other hand if
+		 * the changes are happening right now, just read current value
+		 * directly.
+		 */
+		seqcnt = read_seqcount_begin(&q->ave_seqcnt);
+		ave_nr_running = do_avg_nr_running(q);
+		if (read_seqcount_retry(&q->ave_seqcnt, seqcnt)) {
+			read_seqcount_begin(&q->ave_seqcnt);
+			ave_nr_running = q->ave_nr_running;
+		}
+
+		sum += ave_nr_running;
+	}
+
+	return sum;
+}
 
 /*
  * Global load-average calculations
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index aa12450076cd..b75fd8944ecc 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -418,6 +418,7 @@ struct rq {
 	/* time-based average load */
 	u64 nr_last_stamp;
 	unsigned int ave_nr_running;
+	seqcount_t ave_seqcnt;
 
 	/* capture load from *all* tasks on this cpu: */
 	struct load_weight load;
@@ -1087,25 +1088,30 @@ static inline u64 steal_ticks(u64 steal)
 #define NR_AVE_PERIOD		(1 << NR_AVE_PERIOD_EXP)
 #define NR_AVE_DIV_PERIOD(x)	((x) >> NR_AVE_PERIOD_EXP)
 
-static inline void do_avg_nr_running(struct rq *rq)
+static inline unsigned int do_avg_nr_running(struct rq *rq)
 {
 	s64 nr, deltax;
+	unsigned int ave_nr_running = rq->ave_nr_running;
 
 	deltax = rq->clock_task - rq->nr_last_stamp;
-	rq->nr_last_stamp = rq->clock_task;
 	nr = NR_AVE_SCALE(rq->nr_running);
 
 	if (deltax > NR_AVE_PERIOD)
-		rq->ave_nr_running = nr;
+		ave_nr_running = nr;
 	else
-		rq->ave_nr_running +=
-			NR_AVE_DIV_PERIOD(deltax * (nr - rq->ave_nr_running));
+		ave_nr_running +=
+			NR_AVE_DIV_PERIOD(deltax * (nr - ave_nr_running));
+
+	return ave_nr_running;
 }
 
 static inline void inc_nr_running(struct rq *rq)
 {
-	do_avg_nr_running(rq);
+	write_seqcount_begin(&rq->ave_seqcnt);
+	rq->ave_nr_running = do_avg_nr_running(rq);
+	rq->nr_last_stamp = rq->clock_task;
 	rq->nr_running++;
+	write_seqcount_end(&rq->ave_seqcnt);
 
 #ifdef CONFIG_NO_HZ_FULL
 	if (rq->nr_running == 2) {
@@ -1120,18 +1126,11 @@ static inline void inc_nr_running(struct rq *rq)
 
 static inline void dec_nr_running(struct rq *rq)
 {
-	do_avg_nr_running(rq);
+	write_seqcount_begin(&rq->ave_seqcnt);
+	rq->ave_nr_running = do_avg_nr_running(rq);
+	rq->nr_last_stamp = rq->clock_task;
 	rq->nr_running--;
-}
-
-unsigned long avg_nr_running(void)
-{
-	unsigned long i, sum = 0;
-
-	for_each_online_cpu(i)
-		sum += cpu_rq(i)->ave_nr_running;
-
-	return sum;
+	write_seqcount_end(&rq->ave_seqcnt);
 }
 
 static inline void rq_last_tick_reset(struct rq *rq)