1 files changed, 192 insertions, 37 deletions

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 9ff717d93653..739ec3b359b0 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -29,6 +29,7 @@
 #include <asm/esr.h>
 #include <asm/irq.h>
 #include <asm/memory.h>
+#include <asm/mmu.h>
 #include <asm/ptrace.h>
 #include <asm/thread_info.h>
 #include <asm/uaccess.h>
@@ -70,8 +71,31 @@
 #define BAD_FIQ         2
 #define BAD_ERROR       3
 
-        .macro  kernel_entry, el, regsize = 64
+        .macro kernel_ventry, el, label, regsize = 64
+        .align 7
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+alternative_if ARM64_UNMAP_KERNEL_AT_EL0
+        .if     \el == 0
+        .if     \regsize == 64
+        mrs     x30, tpidrro_el0
+        msr     tpidrro_el0, xzr
+        .else
+        mov     x30, xzr
+        .endif
+        .endif
+alternative_else_nop_endif
+#endif
+
         sub     sp, sp, #S_FRAME_SIZE
+        b       el\()\el\()_\label
+        .endm
+
+        .macro tramp_alias, dst, sym
+        mov_q   \dst, TRAMP_VALIAS
+        add     \dst, \dst, #(\sym - .entry.tramp.text)
+        .endm
+
+        .macro  kernel_entry, el, regsize = 64
         .if     \regsize == 32
         mov     w0, w0                          // zero upper 32 bits of x0
         .endif
@@ -141,7 +165,7 @@ alternative_else_nop_endif
 
         .if     \el != 0
         mrs     x21, ttbr0_el1
-        tst     x21, #0xffff << 48              // Check for the reserved ASID
+        tst     x21, #TTBR_ASID_MASK            // Check for the reserved ASID
         orr     x23, x23, #PSR_PAN_BIT          // Set the emulated PAN in the saved SPSR
         b.eq    1f                              // TTBR0 access already disabled
         and     x23, x23, #~PSR_PAN_BIT         // Clear the emulated PAN in the saved SPSR
@@ -208,7 +232,7 @@ alternative_else_nop_endif
         tbnz    x22, #22, 1f                    // Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
         .endif
 
-        __uaccess_ttbr0_enable x0
+        __uaccess_ttbr0_enable x0, x1
 
         .if     \el == 0
         /*
@@ -217,7 +241,7 @@ alternative_else_nop_endif
          * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
          * corruption).
          */
-        post_ttbr0_update_workaround
+        bl      post_ttbr_update_workaround
         .endif
 1:
         .if     \el != 0
@@ -229,24 +253,20 @@ alternative_else_nop_endif
         .if     \el == 0
         ldr     x23, [sp, #S_SP]                // load return stack pointer
         msr     sp_el0, x23
+        tst     x22, #PSR_MODE32_BIT            // native task?
+        b.eq    3f
+
 #ifdef CONFIG_ARM64_ERRATUM_845719
-alternative_if_not ARM64_WORKAROUND_845719
-        nop
-        nop
-#ifdef CONFIG_PID_IN_CONTEXTIDR
-        nop
-#endif
-alternative_else
-        tbz     x22, #4, 1f
+alternative_if ARM64_WORKAROUND_845719
 #ifdef CONFIG_PID_IN_CONTEXTIDR
         mrs     x29, contextidr_el1
         msr     contextidr_el1, x29
 #else
         msr contextidr_el1, xzr
 #endif
-1:
-alternative_endif
+alternative_else_nop_endif
 #endif
+3:
         .endif
 
         msr     elr_el1, x21                    // set up the return data
@@ -268,7 +288,21 @@ alternative_endif
         ldp     x28, x29, [sp, #16 * 14]
         ldr     lr, [sp, #S_LR]
         add     sp, sp, #S_FRAME_SIZE           // restore sp
-        eret                                    // return to kernel
+
+        .if     \el == 0
+alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+        bne     4f
+        msr     far_el1, x30
+        tramp_alias     x30, tramp_exit_native
+        br      x30
+4:
+        tramp_alias     x30, tramp_exit_compat
+        br      x30
+#endif
+        .else
+        eret
+        .endif
         .endm
 
         .macro  irq_stack_entry
@@ -346,31 +380,31 @@ tsk	.req	x28		// current thread_info
 
         .align  11
 ENTRY(vectors)
-        ventry  el1_sync_invalid                // Synchronous EL1t
-        ventry  el1_irq_invalid                 // IRQ EL1t
-        ventry  el1_fiq_invalid                 // FIQ EL1t
-        ventry  el1_error_invalid               // Error EL1t
+        kernel_ventry   1, sync_invalid                 // Synchronous EL1t
+        kernel_ventry   1, irq_invalid                  // IRQ EL1t
+        kernel_ventry   1, fiq_invalid                  // FIQ EL1t
+        kernel_ventry   1, error_invalid                // Error EL1t
 
-        ventry  el1_sync                        // Synchronous EL1h
-        ventry  el1_irq                         // IRQ EL1h
-        ventry  el1_fiq_invalid                 // FIQ EL1h
-        ventry  el1_error_invalid               // Error EL1h
+        kernel_ventry   1, sync                         // Synchronous EL1h
+        kernel_ventry   1, irq                          // IRQ EL1h
+        kernel_ventry   1, fiq_invalid                  // FIQ EL1h
+        kernel_ventry   1, error_invalid                // Error EL1h
 
-        ventry  el0_sync                        // Synchronous 64-bit EL0
-        ventry  el0_irq                         // IRQ 64-bit EL0
-        ventry  el0_fiq_invalid                 // FIQ 64-bit EL0
-        ventry  el0_error_invalid               // Error 64-bit EL0
+        kernel_ventry   0, sync                         // Synchronous 64-bit EL0
+        kernel_ventry   0, irq                          // IRQ 64-bit EL0
+        kernel_ventry   0, fiq_invalid                  // FIQ 64-bit EL0
+        kernel_ventry   0, error_invalid                // Error 64-bit EL0
 
 #ifdef CONFIG_COMPAT
-        ventry  el0_sync_compat                 // Synchronous 32-bit EL0
-        ventry  el0_irq_compat                  // IRQ 32-bit EL0
-        ventry  el0_fiq_invalid_compat          // FIQ 32-bit EL0
-        ventry  el0_error_invalid_compat        // Error 32-bit EL0
+        kernel_ventry   0, sync_compat, 32              // Synchronous 32-bit EL0
+        kernel_ventry   0, irq_compat, 32               // IRQ 32-bit EL0
+        kernel_ventry   0, fiq_invalid_compat, 32       // FIQ 32-bit EL0
+        kernel_ventry   0, error_invalid_compat, 32     // Error 32-bit EL0
 #else
-        ventry  el0_sync_invalid                // Synchronous 32-bit EL0
-        ventry  el0_irq_invalid                 // IRQ 32-bit EL0
-        ventry  el0_fiq_invalid                 // FIQ 32-bit EL0
-        ventry  el0_error_invalid               // Error 32-bit EL0
+        kernel_ventry   0, sync_invalid, 32             // Synchronous 32-bit EL0
+        kernel_ventry   0, irq_invalid, 32              // IRQ 32-bit EL0
+        kernel_ventry   0, fiq_invalid, 32              // FIQ 32-bit EL0
+        kernel_ventry   0, error_invalid, 32            // Error 32-bit EL0
 #endif
 END(vectors)
 
@@ -572,7 +606,7 @@ el0_sync:
         cmp     x24, #ESR_ELx_EC_FP_EXC64       // FP/ASIMD exception
         b.eq    el0_fpsimd_exc
         cmp     x24, #ESR_ELx_EC_SYS64          // configurable trap
-        b.eq    el0_undef
+        b.eq    el0_sys
         cmp     x24, #ESR_ELx_EC_SP_ALIGN       // stack alignment exception
         b.eq    el0_sp_pc
         cmp     x24, #ESR_ELx_EC_PC_ALIGN       // pc alignment exception
@@ -700,6 +734,16 @@ el0_undef:
         mov     x0, sp
         bl      do_undefinstr
         b       ret_to_user
+el0_sys:
+        /*
+         * System instructions, for trapped cache maintenance instructions
+         */
+        enable_dbg_and_irq
+        ct_user_exit
+        mov     x0, x25
+        mov     x1, sp
+        bl      do_sysinstr
+        b       ret_to_user
 el0_dbg:
         /*
          * Debug exception handling
@@ -911,7 +955,118 @@ __ni_sys_trace:
         bl      do_ni_syscall
         b       __sys_trace_return
 
-        .popsection                             // .entry.text
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+/*
+ * Exception vectors trampoline.
+ */
+        .pushsection ".entry.tramp.text", "ax"
+
+        .macro tramp_map_kernel, tmp
+        mrs     \tmp, ttbr1_el1
+        sub     \tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
+        bic     \tmp, \tmp, #USER_ASID_FLAG
+        msr     ttbr1_el1, \tmp
+#ifdef CONFIG_ARCH_MSM8996
+        /* ASID already in \tmp[63:48] */
+        movk    \tmp, #:abs_g2_nc:(TRAMP_VALIAS >> 12)
+        movk    \tmp, #:abs_g1_nc:(TRAMP_VALIAS >> 12)
+        /* 2MB boundary containing the vectors, so we nobble the walk cache */
+        movk    \tmp, #:abs_g0_nc:((TRAMP_VALIAS & ~(SZ_2M - 1)) >> 12)
+        isb
+        tlbi    vae1, \tmp
+        dsb     nsh
+#endif /* CONFIG_ARCH_MSM8996 */
+        .endm
+
+        .macro tramp_unmap_kernel, tmp
+        mrs     \tmp, ttbr1_el1
+        add     \tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
+        orr     \tmp, \tmp, #USER_ASID_FLAG
+        msr     ttbr1_el1, \tmp
+        /*
+         * We avoid running the post_ttbr_update_workaround here because the
+         * user and kernel ASIDs don't have conflicting mappings, so any
+         * "blessing" as described in:
+         *
+         *   http://lkml.kernel.org/r/56BB848A.6060603@caviumnetworks.com
+         *
+         * will not hurt correctness. Whilst this may partially defeat the
+         * point of using split ASIDs in the first place, it avoids
+         * the hit of invalidating the entire I-cache on every return to
+         * userspace.
+         */
+        .endm
+
+        .macro tramp_ventry, regsize = 64
+        .align  7
+1:
+        .if     \regsize == 64
+        msr     tpidrro_el0, x30        // Restored in kernel_ventry
+        .endif
+        bl      2f
+        b       .
+2:
+        tramp_map_kernel        x30
+#ifdef CONFIG_RANDOMIZE_BASE
+        adr     x30, tramp_vectors + PAGE_SIZE
+#ifndef CONFIG_ARCH_MSM8996
+        isb
+#endif
+        ldr     x30, [x30]
+#else
+        ldr     x30, =vectors
+#endif
+        prfm    plil1strm, [x30, #(1b - tramp_vectors)]
+        msr     vbar_el1, x30
+        add     x30, x30, #(1b - tramp_vectors)
+        isb
+        ret
+        .endm
+
+        .macro tramp_exit, regsize = 64
+        adr     x30, tramp_vectors
+        msr     vbar_el1, x30
+        tramp_unmap_kernel      x30
+        .if     \regsize == 64
+        mrs     x30, far_el1
+        .endif
+        eret
+        .endm
+
+        .align  11
+ENTRY(tramp_vectors)
+        .space  0x400
+
+        tramp_ventry
+        tramp_ventry
+        tramp_ventry
+        tramp_ventry
+
+        tramp_ventry    32
+        tramp_ventry    32
+        tramp_ventry    32
+        tramp_ventry    32
+END(tramp_vectors)
+
+ENTRY(tramp_exit_native)
+        tramp_exit
+END(tramp_exit_native)
+
+ENTRY(tramp_exit_compat)
+        tramp_exit      32
+END(tramp_exit_compat)
+
+        .ltorg
+        .popsection                             // .entry.tramp.text
+#ifdef CONFIG_RANDOMIZE_BASE
+        .pushsection ".rodata", "a"
+        .align PAGE_SHIFT
+        .globl  __entry_tramp_data_start
+__entry_tramp_data_start:
+        .quad   vectors
+        .popsection                             // .rodata
+#endif /* CONFIG_RANDOMIZE_BASE */
+#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
 
 /*
  * Special system call wrappers.