MIPS: Check for accesses beyond the end of the PGD.

For some combinations of PAGE_SIZE and vmbits, it is possible to have
userspace access that are beyond what is covered by the PGD, but within
vmbits.  Such an access would cause the TLB refill handler to load garbage
values for PMD and PTE potentially giving userspace access to parts of the
physical address space to which it is not entitled.

In the TLB refill hot path, we add a single dsrl instruction so we can
check if any bits outside of the range covered by the PGD are set.  In
the vmalloc side we then separate the bad case from the normal vmalloc
case and call tlb_do_page_fault_0 if warranted.  This slows us down a
bit, but has the benefit of yielding deterministic behavior.

[Ralf: Fixed build error for 32-bit kernels.]
[Ralf: Folded lmo commit c8c0e22b2aa3982852b44279638ef37f9aa31b7d into this
 commit.]

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
To: linux-mips@linux-mips.org
Patchwork: http://patchwork.linux-mips.org/patch/1152/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

---

arch/mips/mm/tlbex.c

@@ -31,6 +31,16 @@
 #include <asm/war.h>
 #include <asm/uasm.h>
 
+/*
+ * TLB load/store/modify handlers.
+ *
+ * Only the fastpath gets synthesized at runtime, the slowpath for
+ * do_page_fault remains normal asm.
+ */
+extern void tlb_do_page_fault_0(void);
+extern void tlb_do_page_fault_1(void);
+
+
 static inline int r45k_bvahwbug(void)
 {
 	/* XXX: We should probe for the presence of this bug, but we don't. */
@@ -83,6 +93,7 @@ enum label_id {
 	label_nopage_tlbm,
 	label_smp_pgtable_change,
 	label_r3000_write_probe_fail,
+	label_large_segbits_fault,
 #ifdef CONFIG_HUGETLB_PAGE
 	label_tlb_huge_update,
 #endif
@@ -101,6 +112,7 @@ UASM_L_LA(_nopage_tlbs)
 UASM_L_LA(_nopage_tlbm)
 UASM_L_LA(_smp_pgtable_change)
 UASM_L_LA(_r3000_write_probe_fail)
+UASM_L_LA(_large_segbits_fault)
 #ifdef CONFIG_HUGETLB_PAGE
 UASM_L_LA(_tlb_huge_update)
 #endif
@@ -157,6 +169,10 @@ static u32 tlb_handler[128] __cpuinitdata;
 static struct uasm_label labels[128] __cpuinitdata;
 static struct uasm_reloc relocs[128] __cpuinitdata;
 
+#ifdef CONFIG_64BIT
+static int check_for_high_segbits __cpuinitdata;
+#endif
+
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 /*
  * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
@@ -532,7 +548,24 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 	 * The vmalloc handling is not in the hotpath.
 	 */
 	uasm_i_dmfc0(p, tmp, C0_BADVADDR);
-	uasm_il_bltz(p, r, tmp, label_vmalloc);
+
+	if (check_for_high_segbits) {
+		/*
+		 * The kernel currently implicitely assumes that the
+		 * MIPS SEGBITS parameter for the processor is
+		 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
+		 * allocate virtual addresses outside the maximum
+		 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
+		 * that doesn't prevent user code from accessing the
+		 * higher xuseg addresses.  Here, we make sure that
+		 * everything but the lower xuseg addresses goes down
+		 * the module_alloc/vmalloc path.
+		 */
+		uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
+		uasm_il_bnez(p, r, ptr, label_vmalloc);
+	} else {
+		uasm_il_bltz(p, r, tmp, label_vmalloc);
+	}
 	/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
 
 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT
@@ -583,28 +616,64 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 #endif
 }
 
+enum vmalloc64_mode {not_refill, refill};
 /*
  * BVADDR is the faulting address, PTR is scratch.
  * PTR will hold the pgd for vmalloc.
  */
 static void __cpuinit
 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
-			unsigned int bvaddr, unsigned int ptr)
+			unsigned int bvaddr, unsigned int ptr,
+			enum vmalloc64_mode mode)
 {
 	long swpd = (long)swapper_pg_dir;
+	int single_insn_swpd;
+	int did_vmalloc_branch = 0;
+
+	single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
 
 	uasm_l_vmalloc(l, *p);
 
-	if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
+	if (mode == refill && check_for_high_segbits) {
-		uasm_il_b(p, r, label_vmalloc_done);
+		if (single_insn_swpd) {
-		uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
+			uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
-	} else {
+			uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
-		UASM_i_LA_mostly(p, ptr, swpd);
+			did_vmalloc_branch = 1;
-		uasm_il_b(p, r, label_vmalloc_done);
+			/* fall through */
-		if (uasm_in_compat_space_p(swpd))
+		} else {
-			uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
+			uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
-		else
+		}
-			uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
+	}
+	if (!did_vmalloc_branch) {
+		if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
+			uasm_il_b(p, r, label_vmalloc_done);
+			uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
+		} else {
+			UASM_i_LA_mostly(p, ptr, swpd);
+			uasm_il_b(p, r, label_vmalloc_done);
+			if (uasm_in_compat_space_p(swpd))
+				uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
+			else
+				uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
+		}
+	}
+	if (mode == refill && check_for_high_segbits) {
+		uasm_l_large_segbits_fault(l, *p);
+		/*
+		 * We get here if we are an xsseg address, or if we are
+		 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
+		 *
+		 * Ignoring xsseg (assume disabled so would generate
+		 * (address errors?), the only remaining possibility
+		 * is the upper xuseg addresses.  On processors with
+		 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
+		 * addresses would have taken an address error. We try
+		 * to mimic that here by taking a load/istream page
+		 * fault.
+		 */
+		UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
+		uasm_i_jr(p, ptr);
+		uasm_i_nop(p);
 	}
 }
 
@@ -823,7 +892,7 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
 #endif
 
 #ifdef CONFIG_64BIT
-	build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
+	build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, refill);
 #endif
 
 	/*
@@ -932,15 +1001,6 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
 	dump_handler((u32 *)ebase, 64);
 }
 
-/*
- * TLB load/store/modify handlers.
- *
- * Only the fastpath gets synthesized at runtime, the slowpath for
- * do_page_fault remains normal asm.
- */
-extern void tlb_do_page_fault_0(void);
-extern void tlb_do_page_fault_1(void);
-
 /*
  * 128 instructions for the fastpath handler is generous and should
  * never be exceeded.
@@ -1300,7 +1360,7 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
 	uasm_i_eret(p); /* return from trap */
 
 #ifdef CONFIG_64BIT
-	build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
+	build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
 #endif
 }
 
@@ -1524,6 +1584,10 @@ void __cpuinit build_tlb_refill_handler(void)
 	 */
 	static int run_once = 0;
 
+#ifdef CONFIG_64BIT
+	check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
+#endif
+
 	switch (current_cpu_type()) {
 	case CPU_R2000:
 	case CPU_R3000: