1 files changed, 116 insertions, 38 deletions

diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
index 367a954f9937..1c694f3c643c 100644
--- a/arch/arm64/kernel/hw_breakpoint.c
+++ b/arch/arm64/kernel/hw_breakpoint.c
@@ -36,6 +36,7 @@
 #include <asm/traps.h>
 #include <asm/cputype.h>
 #include <asm/system_misc.h>
+#include <asm/uaccess.h>
 
 /* Breakpoint currently in use for each BRP. */
 static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
@@ -317,9 +318,21 @@ static int get_hbp_len(u8 hbp_len)
         case ARM_BREAKPOINT_LEN_2:
                 len_in_bytes = 2;
                 break;
+        case ARM_BREAKPOINT_LEN_3:
+                len_in_bytes = 3;
+                break;
         case ARM_BREAKPOINT_LEN_4:
                 len_in_bytes = 4;
                 break;
+        case ARM_BREAKPOINT_LEN_5:
+                len_in_bytes = 5;
+                break;
+        case ARM_BREAKPOINT_LEN_6:
+                len_in_bytes = 6;
+                break;
+        case ARM_BREAKPOINT_LEN_7:
+                len_in_bytes = 7;
+                break;
         case ARM_BREAKPOINT_LEN_8:
                 len_in_bytes = 8;
                 break;
@@ -349,7 +362,7 @@ int arch_check_bp_in_kernelspace(struct perf_event *bp)
  * to generic breakpoint descriptions.
  */
 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
-                           int *gen_len, int *gen_type)
+                           int *gen_len, int *gen_type, int *offset)
 {
         /* Type */
         switch (ctrl.type) {
@@ -369,17 +382,33 @@ int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
                 return -EINVAL;
         }
 
+        if (!ctrl.len)
+                return -EINVAL;
+        *offset = __ffs(ctrl.len);
+
         /* Len */
-        switch (ctrl.len) {
+        switch (ctrl.len >> *offset) {
         case ARM_BREAKPOINT_LEN_1:
                 *gen_len = HW_BREAKPOINT_LEN_1;
                 break;
         case ARM_BREAKPOINT_LEN_2:
                 *gen_len = HW_BREAKPOINT_LEN_2;
                 break;
+        case ARM_BREAKPOINT_LEN_3:
+                *gen_len = HW_BREAKPOINT_LEN_3;
+                break;
         case ARM_BREAKPOINT_LEN_4:
                 *gen_len = HW_BREAKPOINT_LEN_4;
                 break;
+        case ARM_BREAKPOINT_LEN_5:
+                *gen_len = HW_BREAKPOINT_LEN_5;
+                break;
+        case ARM_BREAKPOINT_LEN_6:
+                *gen_len = HW_BREAKPOINT_LEN_6;
+                break;
+        case ARM_BREAKPOINT_LEN_7:
+                *gen_len = HW_BREAKPOINT_LEN_7;
+                break;
         case ARM_BREAKPOINT_LEN_8:
                 *gen_len = HW_BREAKPOINT_LEN_8;
                 break;
@@ -423,9 +452,21 @@ static int arch_build_bp_info(struct perf_event *bp)
         case HW_BREAKPOINT_LEN_2:
                 info->ctrl.len = ARM_BREAKPOINT_LEN_2;
                 break;
+        case HW_BREAKPOINT_LEN_3:
+                info->ctrl.len = ARM_BREAKPOINT_LEN_3;
+                break;
         case HW_BREAKPOINT_LEN_4:
                 info->ctrl.len = ARM_BREAKPOINT_LEN_4;
                 break;
+        case HW_BREAKPOINT_LEN_5:
+                info->ctrl.len = ARM_BREAKPOINT_LEN_5;
+                break;
+        case HW_BREAKPOINT_LEN_6:
+                info->ctrl.len = ARM_BREAKPOINT_LEN_6;
+                break;
+        case HW_BREAKPOINT_LEN_7:
+                info->ctrl.len = ARM_BREAKPOINT_LEN_7;
+                break;
         case HW_BREAKPOINT_LEN_8:
                 info->ctrl.len = ARM_BREAKPOINT_LEN_8;
                 break;
@@ -517,18 +558,17 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp)
                 default:
                         return -EINVAL;
                 }
-
-                info->address &= ~alignment_mask;
-                info->ctrl.len <<= offset;
         } else {
                 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE)
                         alignment_mask = 0x3;
                 else
                         alignment_mask = 0x7;
-                if (info->address & alignment_mask)
-                        return -EINVAL;
+                offset = info->address & alignment_mask;
         }
 
+        info->address &= ~alignment_mask;
+        info->ctrl.len <<= offset;
+
         /*
          * Disallow per-task kernel breakpoints since these would
          * complicate the stepping code.
@@ -661,12 +701,47 @@ unlock:
 }
 NOKPROBE_SYMBOL(breakpoint_handler);
 
+/*
+ * Arm64 hardware does not always report a watchpoint hit address that matches
+ * one of the watchpoints set. It can also report an address "near" the
+ * watchpoint if a single instruction access both watched and unwatched
+ * addresses. There is no straight-forward way, short of disassembling the
+ * offending instruction, to map that address back to the watchpoint. This
+ * function computes the distance of the memory access from the watchpoint as a
+ * heuristic for the likelyhood that a given access triggered the watchpoint.
+ *
+ * See Section D2.10.5 "Determining the memory location that caused a Watchpoint
+ * exception" of ARMv8 Architecture Reference Manual for details.
+ *
+ * The function returns the distance of the address from the bytes watched by
+ * the watchpoint. In case of an exact match, it returns 0.
+ */
+static u64 get_distance_from_watchpoint(unsigned long addr, u64 val,
+                                        struct arch_hw_breakpoint_ctrl *ctrl)
+{
+        u64 wp_low, wp_high;
+        u32 lens, lene;
+
+        lens = __ffs(ctrl->len);
+        lene = __fls(ctrl->len);
+
+        wp_low = val + lens;
+        wp_high = val + lene;
+        if (addr < wp_low)
+                return wp_low - addr;
+        else if (addr > wp_high)
+                return addr - wp_high;
+        else
+                return 0;
+}
+
 static int watchpoint_handler(unsigned long addr, unsigned int esr,
                               struct pt_regs *regs)
 {
-        int i, step = 0, *kernel_step, access;
+        int i, step = 0, *kernel_step, access, closest_match = 0;
+        u64 min_dist = -1, dist;
         u32 ctrl_reg;
-        u64 val, alignment_mask;
+        u64 val;
         struct perf_event *wp, **slots;
         struct debug_info *debug_info;
         struct arch_hw_breakpoint *info;
@@ -675,35 +750,15 @@ static int watchpoint_handler(unsigned long addr, unsigned int esr,
         slots = this_cpu_ptr(wp_on_reg);
         debug_info = &current->thread.debug;
 
+        /*
+         * Find all watchpoints that match the reported address. If no exact
+         * match is found. Attribute the hit to the closest watchpoint.
+         */
+        rcu_read_lock();
         for (i = 0; i < core_num_wrps; ++i) {
-                rcu_read_lock();
-
                 wp = slots[i];
-
                 if (wp == NULL)
-                        goto unlock;
-
-                info = counter_arch_bp(wp);
-                /* AArch32 watchpoints are either 4 or 8 bytes aligned. */
-                if (is_compat_task()) {
-                        if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
-                                alignment_mask = 0x7;
-                        else
-                                alignment_mask = 0x3;
-                } else {
-                        alignment_mask = 0x7;
-                }
-
-                /* Check if the watchpoint value matches. */
-                val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
-                if (val != (addr & ~alignment_mask))
-                        goto unlock;
-
-                /* Possible match, check the byte address select to confirm. */
-                ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
-                decode_ctrl_reg(ctrl_reg, &ctrl);
-                if (!((1 << (addr & alignment_mask)) & ctrl.len))
-                        goto unlock;
+                        continue;
 
                 /*
                  * Check that the access type matches.
@@ -712,18 +767,41 @@ static int watchpoint_handler(unsigned long addr, unsigned int esr,
                 access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W :
                          HW_BREAKPOINT_R;
                 if (!(access & hw_breakpoint_type(wp)))
-                        goto unlock;
+                        continue;
 
+                /* Check if the watchpoint value and byte select match. */
+                val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
+                ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
+                decode_ctrl_reg(ctrl_reg, &ctrl);
+                dist = get_distance_from_watchpoint(addr, val, &ctrl);
+                if (dist < min_dist) {
+                        min_dist = dist;
+                        closest_match = i;
+                }
+                /* Is this an exact match? */
+                if (dist != 0)
+                        continue;
+
+                info = counter_arch_bp(wp);
                 info->trigger = addr;
                 perf_bp_event(wp, regs);
 
                 /* Do we need to handle the stepping? */
                 if (!wp->overflow_handler)
                         step = 1;
+        }
+        if (min_dist > 0 && min_dist != -1) {
+                /* No exact match found. */
+                wp = slots[closest_match];
+                info = counter_arch_bp(wp);
+                info->trigger = addr;
+                perf_bp_event(wp, regs);
 
-unlock:
-                rcu_read_unlock();
+                /* Do we need to handle the stepping? */
+                if (!wp->overflow_handler)
+                        step = 1;
         }
+        rcu_read_unlock();
 
         if (!step)
                 return 0;