ARM: tegra: clock: Dynamically re-lock memory pll

So far Tegra3 EMC DFS allowed only scaling rates that can be divided
down from two fixed rate plls: memory PLLM, and peripheral PLLP. PLLM
is always running at maximum SDRAM rate set at boot time, while PLLP
rate 408MHz is fixed across all Tegra3 platforms.

This commit implements dynamic re-locking of PLLM at run time. Now
memory pll can lock either at boot rate or additional auxiliary rate
that is selected as follows: auxiliary PLLM rate must be present in
EMC DFS table, it must exactly match one of the rate steps for Tegra3
graphics bus with PLLC clock source (cbus), and must not be a proper
factor of boot PLLM rate or PLLP fixed rate.

When switching PLLM between boot and auxiliary rate, PLLC is used as
backup memory pll, and during this time cbus is locked at auxiliary
rate. In addition system bus is forced to temporarily use PLLP as
a clock source (this is necessary as sbus main clock source is PLLM
secondary divider PLLM_OUT1).

Limitations:
- only one auxiliary rate is supported, and it should be below PLLM
boot rate, but above half of boot rate
- dynamic re-lock is allowed only on LPDDR2 platforms
- no clock other than EMC and system bus could use PLLM as a source;
so for dynamic re-lock to work CONFIG_TEGRA_PLLM_RESTRICTED must be
selected, and VI clock (not covered by PLLM restricted configuration)
must be moved to PLLP.

Bug 1005576

Change-Id: I6177107c89c3cbe975a1d940927efa1ed0ea61ec
Signed-off-by: Alex Frid <afrid@nvidia.com>
Reviewed-on: http://git-master/r/111438
Reviewed-by: Rohan Somvanshi <rsomvanshi@nvidia.com>
Tested-by: Rohan Somvanshi <rsomvanshi@nvidia.com>
(cherry picked from commit dc4d468a6acabfb268e7a7f44b45bb7354e9a99a)
Reviewed-on: http://git-master/r/114760
Reviewed-by: Automatic_Commit_Validation_User
Reviewed-by: Jihoon Bang <jbang@nvidia.com>
Reviewed-by: Yu-Huan Hsu <yhsu@nvidia.com>

3 files changed, 171 insertions, 16 deletions

diff --git a/arch/arm/mach-tegra/clock.h b/arch/arm/mach-tegra/clock.h
index 8af1e1b21787..c27176b1cc0d 100644
--- a/arch/arm/mach-tegra/clock.h
+++ b/arch/arm/mach-tegra/clock.h
@@ -255,9 +255,12 @@ long tegra_emc_round_rate(unsigned long rate);
 struct clk *tegra_emc_predict_parent(unsigned long rate, u32 *div_value);
 void tegra_emc_timing_invalidate(void);
 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
+static inline int tegra_emc_backup(unsigned long rate)
+{ return 0; }
 static inline bool tegra_clk_is_parent_allowed(struct clk *c, struct clk *p)
 { return true; }
 #else
+int tegra_emc_backup(unsigned long rate);
 bool tegra_clk_is_parent_allowed(struct clk *c, struct clk *p);
 #endif
 
diff --git a/arch/arm/mach-tegra/tegra3_clocks.c b/arch/arm/mach-tegra/tegra3_clocks.c
index ca8cd5c449b9..1161f1ca64f9 100644
--- a/arch/arm/mach-tegra/tegra3_clocks.c
+++ b/arch/arm/mach-tegra/tegra3_clocks.c
@@ -314,6 +314,8 @@
 
 static void tegra3_pllp_init_dependencies(unsigned long pllp_rate);
 static int tegra3_clk_shared_bus_update(struct clk *bus);
+static int tegra3_emc_relock_set_rate(struct clk *emc, unsigned long old_rate,
+	unsigned long new_rate, unsigned long new_pll_rate);
 
 static unsigned long cpu_stay_on_backup_max;
 static struct clk *emc_bridge;
@@ -2636,8 +2638,15 @@ static int tegra3_emc_clk_set_rate(struct clk *c, unsigned long rate)
 	 * to achieve requested rate. */
 	p = tegra_emc_predict_parent(rate, &div_value);
 	div_value += 2;		/* emc has fractional DIV_U71 divider */
+
+	/* No matching rate in emc dfs table */
+	if (IS_ERR(p))
+		return PTR_ERR(p);
+
+	/* Table rate found, but need to relock source pll */
 	if (!p)
-		return -EINVAL;
+		return tegra3_emc_relock_set_rate(c, clk_get_rate_locked(c),
+						  rate, rate * (div_value / 2));
 
 	if (p == c->parent) {
 		if (div_value == c->div)
@@ -4076,7 +4085,7 @@ static struct clk_mux_sel mux_pllm_pllc_pllp_plla[] = {
 
 static struct clk_mux_sel mux_pllm_pllc_pllp_clkm[] = {
 	{ .input = &tegra_pll_m, .value = 0},
-	/* { .input = &tegra_pll_c, .value = 1}, not used on tegra3 */
+	{ .input = &tegra_pll_c, .value = 1},
 	{ .input = &tegra_pll_p, .value = 2},
 	{ .input = &tegra_clk_m, .value = 3},
 	{ 0, 0},
@@ -4200,6 +4209,9 @@ static struct clk tegra_clk_emc = {
 	.u.periph = {
 		.clk_num = 57,
 	},
+	.shared_bus_backup = {
+		.input = &tegra_pll_c,
+	},
 	.rate_change_nh = &emc_rate_change_nh,
 };
 
@@ -4526,6 +4538,86 @@ struct clk *tegra_ptr_clks[] = {
 	&tegra_clk_cbus,
 };
 
+static int tegra3_emc_relock_set_rate(struct clk *emc, unsigned long old_rate,
+	unsigned long new_rate, unsigned long new_pll_rate)
+{
+	int ret;
+
+	struct clk *sbus = &tegra_clk_sbus_cmplx;
+	struct clk *cbus = &tegra_clk_cbus;
+	struct clk *pll_m = &tegra_pll_m;
+	unsigned long backup_rate = emc->shared_bus_backup.bus_rate;
+	unsigned long flags;
+
+	bool on_pllm = emc->parent == pll_m;
+
+	/*
+	 * Relock procedure pre-conditions:
+	 * - LPDDR2 only
+	 * - EMC clock is enabled, and EMC backup rate is found in DFS table
+	 * - All 3 shared buses: emc, sbus, cbus can sleep
+	 */
+	if ((tegra_emc_get_dram_type() != DRAM_TYPE_LPDDR2) || !emc->refcnt ||
+	    !backup_rate || (cbus->parent != emc->shared_bus_backup.input) ||
+	    !clk_cansleep(emc) || !clk_cansleep(cbus) || !clk_cansleep(sbus))
+		return -ENOSYS;
+
+	/* Move sbus from PLLM by setting it at low rate threshold level */
+	clk_lock_save(sbus, &flags);
+	if (clk_get_rate_locked(sbus) > sbus->u.system.threshold) {
+		ret = clk_set_rate_locked(sbus, sbus->u.system.threshold);
+		if (ret)
+			goto _sbus_out;
+	}
+
+	/* If PLLM is current EMC parent set cbus to backup rate, and move EMC
+	   to backup PLLC */
+	if (on_pllm) {
+		clk_lock_save(cbus, &flags);
+		clk_enable(cbus->parent);
+		ret = clk_set_rate_locked(cbus, backup_rate);
+		if (ret) {
+			clk_disable(cbus->parent);
+			goto _cbus_out;
+		}
+
+		ret = tegra_emc_backup(backup_rate);
+		if (ret) {
+			clk_disable(cbus->parent);
+			goto _cbus_out;
+		}
+		clk_disable(emc->parent);
+		clk_reparent(emc, cbus->parent);
+	}
+
+	/*
+	 * Re-lock PLLM and switch EMC to it; relocking error indicates that
+	 * PLLM has some other than EMC or sbus client. In this case PLLM has
+	 * not been changed, and we still can safely switch back. Recursive
+	 * tegra3_emc_clk_set_rate() call below will be resolved, since PLLM
+	 * is now matching target rate.
+	 */
+	ret = clk_set_rate(pll_m, new_pll_rate);
+	if (ret) {
+		if (on_pllm)
+			tegra3_emc_clk_set_rate(emc, old_rate);
+	} else
+		ret = tegra3_emc_clk_set_rate(emc, new_rate);
+
+
+_cbus_out:
+	if (on_pllm) {
+		tegra3_clk_shared_bus_update(cbus);
+		clk_unlock_restore(cbus, &flags);
+	}
+
+_sbus_out:
+	tegra3_clk_shared_bus_update(sbus);
+	clk_unlock_restore(sbus, &flags);
+
+	return ret;
+}
+
 /*
  * Backup rate targets for each CPU mode is selected below Fmax(Vmin), and
  * high enough to avoid voltage droop when CPU clock is switched between
diff --git a/arch/arm/mach-tegra/tegra3_emc.c b/arch/arm/mach-tegra/tegra3_emc.c
index 997507b50843..a138091d9197 100644
--- a/arch/arm/mach-tegra/tegra3_emc.c
+++ b/arch/arm/mach-tegra/tegra3_emc.c
@@ -717,7 +717,7 @@ static inline void emc_cfg_power_restore(void)
  * This function updates the shadow registers for a new clock frequency,
  * and relies on the clock lock on the emc clock to avoid races between
  * multiple frequency changes */
-int tegra_emc_set_rate(unsigned long rate)
+static int emc_set_rate(unsigned long rate, bool use_backup)
 {
 	int i;
 	u32 clk_setting;
@@ -750,7 +750,8 @@ int tegra_emc_set_rate(unsigned long rate)
 	else
 		last_timing = emc_timing;
 
-	clk_setting = tegra_emc_clk_sel[i].value;
+	clk_setting = use_backup ? emc->shared_bus_backup.value :
+		tegra_emc_clk_sel[i].value;
 
 	spin_lock_irqsave(&emc_access_lock, flags);
 	emc_set_clock(&tegra_emc_table[i], last_timing, clk_setting);
@@ -766,6 +767,17 @@ int tegra_emc_set_rate(unsigned long rate)
 	return 0;
 }
 
+int tegra_emc_set_rate(unsigned long rate)
+{
+	return emc_set_rate(rate, false);
+}
+
+int tegra_emc_backup(unsigned long rate)
+{
+	BUG_ON(rate != emc->shared_bus_backup.bus_rate);
+	return emc_set_rate(rate, true);
+}
+
 /* Select the closest EMC rate that is higher than the requested rate */
 long tegra_emc_round_rate(unsigned long rate)
 {
@@ -808,7 +820,7 @@ struct clk *tegra_emc_predict_parent(unsigned long rate, u32 *div_value)
 	int i;
 
 	if (!tegra_emc_table)
-		return NULL;
+		return ERR_PTR(-ENOENT);
 
 	pr_debug("%s: %lu\n", __func__, rate);
 
@@ -817,25 +829,40 @@ struct clk *tegra_emc_predict_parent(unsigned long rate, u32 *div_value)
 
 	for (i = 0; i < tegra_emc_table_size; i++) {
 		if (tegra_emc_table[i].rate == rate) {
+			struct clk *p = tegra_emc_clk_sel[i].input;
+
 			*div_value = (tegra_emc_clk_sel[i].value &
 				EMC_CLK_DIV_MASK) >> EMC_CLK_DIV_SHIFT;
-			return tegra_emc_clk_sel[i].input;
+			if (tegra_emc_clk_sel[i].input_rate != clk_get_rate(p))
+				return NULL;
+
+			return p;
 		}
 	}
-
-	return NULL;
+	return ERR_PTR(-ENOENT);
 }
 
-int find_matching_input(unsigned long table_rate, struct emc_sel *emc_clk_sel)
+int find_matching_input(unsigned long table_rate, bool mc_same_freq,
+			struct emc_sel *emc_clk_sel, struct clk *cbus)
 {
-	u32 div_value;
-	unsigned long input_rate;
+	u32 div_value = 0;
+	unsigned long input_rate = 0;
 	const struct clk_mux_sel *sel;
+	const struct clk_mux_sel *parent_sel = NULL;
+	const struct clk_mux_sel *backup_sel = NULL;
 
 	/* Table entries specify rate in kHz */
 	table_rate *= 1000;
 
 	for (sel = emc->inputs; sel->input != NULL; sel++) {
+		if (sel->input == emc->shared_bus_backup.input) {
+			backup_sel = sel;
+			continue;	/* skip backup souce */
+		}
+
+		if (sel->input == emc->parent)
+			parent_sel = sel;
+
 		input_rate = clk_get_rate(sel->input);
 
 		if ((input_rate >= table_rate) &&
@@ -845,6 +872,37 @@ int find_matching_input(unsigned long table_rate, struct emc_sel *emc_clk_sel)
 		}
 	}
 
+#ifdef CONFIG_TEGRA_PLLM_RESTRICTED
+	/*
+	 * When match not found, check if this rate can be backed-up by cbus
+	 * Then, we will be able to re-lock boot parent PLLM, and use it as
+	 * an undivided source. Backup is supported only on LPDDR2 platforms
+	 * with restricted PLLM usage. Just one backup entry is recognized,
+	 * and it must be between EMC maximum and half maximum rates.
+	 */
+	if ((dram_type == DRAM_TYPE_LPDDR2) && (sel->input == NULL) &&
+	    (emc->shared_bus_backup.bus_rate == 0) && cbus) {
+		BUG_ON(!parent_sel || !backup_sel);
+
+		if ((table_rate == clk_round_rate(cbus, table_rate)) &&
+		    (table_rate < clk_get_max_rate(emc)) &&
+		    (table_rate >= clk_get_max_rate(emc) / 2)) {
+			emc->shared_bus_backup.bus_rate = table_rate;
+
+			/* Get ready emc clock backup selection settings */
+			emc->shared_bus_backup.value =
+				(backup_sel->value << EMC_CLK_SOURCE_SHIFT) |
+				(cbus->div << EMC_CLK_DIV_SHIFT) |
+				(mc_same_freq ? EMC_CLK_MC_SAME_FREQ : 0);
+
+			/* Select undivided PLLM as regular source */
+			sel = parent_sel;
+			input_rate = table_rate;
+			div_value = 0;
+		}
+	}
+#endif
+
 	if (sel->input) {
 		emc_clk_sel->input = sel->input;
 		emc_clk_sel->input_rate = input_rate;
@@ -854,6 +912,8 @@ int find_matching_input(unsigned long table_rate, struct emc_sel *emc_clk_sel)
 		emc_clk_sel->value |= (div_value << EMC_CLK_DIV_SHIFT);
 		if ((div_value == 0) && (emc_clk_sel->input == emc->parent))
 			emc_clk_sel->value |= EMC_CLK_LOW_JITTER_ENABLE;
+		if (mc_same_freq)
+			emc_clk_sel->value |= EMC_CLK_MC_SAME_FREQ;
 		return 0;
 	}
 	return -EINVAL;
@@ -963,6 +1023,7 @@ void tegra_init_emc(const struct tegra_emc_table *table, int table_size)
 	u32 reg;
 	bool max_entry = false;
 	unsigned long boot_rate, max_rate;
+	struct clk *cbus = tegra_get_clock_by_name("cbus");
 
 	emc_stats.clkchange_count = 0;
 	spin_lock_init(&emc_stats.spinlock);
@@ -1006,13 +1067,16 @@ void tegra_init_emc(const struct tegra_emc_table *table, int table_size)
 
 	/* Match EMC source/divider settings with table entries */
 	for (i = 0; i < tegra_emc_table_size; i++) {
+		bool mc_same_freq = MC_EMEM_ARB_MISC0_EMC_SAME_FREQ &
+			table[i].burst_regs[MC_EMEM_ARB_MISC0_INDEX];
 		unsigned long table_rate = table[i].rate;
 		if (!table_rate)
 			continue;
 
 		BUG_ON(table[i].rev != table[0].rev);
 
-		if (find_matching_input(table_rate, &tegra_emc_clk_sel[i]))
+		if (find_matching_input(table_rate, mc_same_freq,
+					&tegra_emc_clk_sel[i], cbus))
 			continue;
 
 		if (table_rate == boot_rate)
@@ -1020,10 +1084,6 @@ void tegra_init_emc(const struct tegra_emc_table *table, int table_size)
 
 		if (table_rate == max_rate)
 			max_entry = true;
-
-		if (table[i].burst_regs[MC_EMEM_ARB_MISC0_INDEX] &
-		    MC_EMEM_ARB_MISC0_EMC_SAME_FREQ)
-			tegra_emc_clk_sel[i].value |= EMC_CLK_MC_SAME_FREQ;
 	}
 
 	/* Validate EMC rate and voltage limits */