From 16f8c2dfc148f67f92bcac35c2fb57f05444ee65 Mon Sep 17 00:00:00 2001 From: Larry Bassel Date: Wed, 15 Oct 2014 15:03:09 -0700 Subject: arm64: Support arch_irq_work_raise() via self IPIs Backport of the following patch to LSK 3.14: commit eb631bb5bf5b042202aaaee4a8dd8f863ba2a900 Author: Larry Bassel Date: Mon May 12 16:48:51 2014 +0100 arm64: Support arch_irq_work_raise() via self IPIs Support for arch_irq_work_raise() was missing from arm64 (a prerequisite for FULL_NOHZ). This patch is based on the arm32 patch ARM 7872/1. commit bf18525fd793101df42a1344ecc48b49b62e48c9 Author: Stephen Boyd Date: Tue Oct 29 20:32:56 2013 +0100 ARM: 7872/1: Support arch_irq_work_raise() via self IPIs By default, IRQ work is run from the tick interrupt (see irq_work_run() in update_process_times()). When we're in full NOHZ mode, restarting the tick requires the use of IRQ work and if the only place we run IRQ work is in the tick interrupt we have an unbreakable cycle. Implement arch_irq_work_raise() via self IPIs to break this cycle and get the tick started again. Note that we implement this via IPIs which are only available on SMP builds. This shouldn't be a problem because full NOHZ is only supported on SMP builds anyway. Signed-off-by: Stephen Boyd Reviewed-by: Kevin Hilman Cc: Frederic Weisbecker Signed-off-by: Russell King Signed-off-by: Larry Bassel Reviewed-by: Kevin Hilman Signed-off-by: Catalin Marinas Signed-off-by: Larry Bassel --- arch/arm64/include/asm/hardirq.h | 2 +- arch/arm64/kernel/smp.c | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h index ae4801d77514..0be67821f9ce 100644 --- a/arch/arm64/include/asm/hardirq.h +++ b/arch/arm64/include/asm/hardirq.h @@ -20,7 +20,7 @@ #include #include -#define NR_IPI 5 +#define NR_IPI 6 typedef struct { unsigned int __softirq_pending; diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 9660750f34ba..6819fab9a80b 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -62,6 +63,7 @@ enum ipi_msg_type { IPI_CALL_FUNC_SINGLE, IPI_CPU_STOP, IPI_TIMER, + IPI_IRQ_WORK, }; /* @@ -454,6 +456,14 @@ void arch_send_call_function_single_ipi(int cpu) smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); } +#ifdef CONFIG_IRQ_WORK +void arch_irq_work_raise(void) +{ + if (smp_cross_call) + smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); +} +#endif + static const char *ipi_types[NR_IPI] = { #define S(x,s) [x - IPI_RESCHEDULE] = s S(IPI_RESCHEDULE, "Rescheduling interrupts"), @@ -461,6 +471,7 @@ static const char *ipi_types[NR_IPI] = { S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"), S(IPI_CPU_STOP, "CPU stop interrupts"), S(IPI_TIMER, "Timer broadcast interrupts"), + S(IPI_IRQ_WORK, "IRQ work interrupts"), }; void show_ipi_list(struct seq_file *p, int prec) @@ -553,6 +564,14 @@ void handle_IPI(int ipinr, struct pt_regs *regs) break; #endif +#ifdef CONFIG_IRQ_WORK + case IPI_IRQ_WORK: + irq_enter(); + irq_work_run(); + irq_exit(); + break; +#endif + default: pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr); break; -- cgit v1.2.3 From 333625b7586d2753a77f32e1f898ab7cc6cf7655 Mon Sep 17 00:00:00 2001 From: Larry Bassel Date: Wed, 15 Oct 2014 15:15:56 -0700 Subject: arm64: adjust el0_sync so that a function can be called Backport of the following patch to 3.14 LSK: commit 6ab6463aeb5fbc75fa3227befb508fc33b34dbf1 Author: Larry Bassel Date: Fri May 30 20:34:14 2014 +0100 arm64: adjust el0_sync so that a function can be called To implement the context tracker properly on arm64, a function call needs to be made after debugging and interrupts are turned on, but before the lr is changed to point to ret_to_user(). If the function call is made after the lr is changed the function will not return to the correct place. For similar reasons, defer the setting of x0 so that it doesn't need to be saved around the function call (save far_el1 in x26 temporarily instead). Acked-by: Will Deacon Reviewed-by: Kevin Hilman Tested-by: Kevin Hilman Signed-off-by: Larry Bassel Signed-off-by: Will Deacon Signed-off-by: Catalin Marinas Signed-off-by: Larry Bassel --- arch/arm64/kernel/entry.S | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 6d26ebc260ad..c779ef357e43 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -348,7 +348,6 @@ el0_sync: lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state b.eq el0_svc - adr lr, ret_from_exception cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 b.eq el0_da cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 @@ -377,7 +376,6 @@ el0_sync_compat: lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state b.eq el0_svc_compat - adr lr, ret_from_exception cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 b.eq el0_da cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 @@ -420,28 +418,31 @@ el0_da: /* * Data abort handling */ - mrs x0, far_el1 - bic x0, x0, #(0xff << 56) + mrs x26, far_el1 disable_step x1 isb enable_dbg // enable interrupts before calling the main handler enable_irq + bic x0, x26, #(0xff << 56) mov x1, x25 mov x2, sp + adr lr, ret_from_exception b do_mem_abort el0_ia: /* * Instruction abort handling */ - mrs x0, far_el1 + mrs x26, far_el1 disable_step x1 isb enable_dbg // enable interrupts before calling the main handler enable_irq + mov x0, x26 orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts mov x2, sp + adr lr, ret_from_exception b do_mem_abort el0_fpsimd_acc: /* @@ -449,6 +450,7 @@ el0_fpsimd_acc: */ mov x0, x25 mov x1, sp + adr lr, ret_from_exception b do_fpsimd_acc el0_fpsimd_exc: /* @@ -456,19 +458,22 @@ el0_fpsimd_exc: */ mov x0, x25 mov x1, sp + adr lr, ret_from_exception b do_fpsimd_exc el0_sp_pc: /* * Stack or PC alignment exception handling */ - mrs x0, far_el1 + mrs x26, far_el1 disable_step x1 isb enable_dbg // enable interrupts before calling the main handler enable_irq + mov x0, x26 mov x1, x25 mov x2, sp + adr lr, ret_from_exception b do_sp_pc_abort el0_undef: /* @@ -477,6 +482,7 @@ el0_undef: mov x0, sp // enable interrupts before calling the main handler enable_irq + adr lr, ret_from_exception b do_undefinstr el0_dbg: /* @@ -492,6 +498,7 @@ el0_inv: mov x0, sp mov x1, #BAD_SYNC mrs x2, esr_el1 + adr lr, ret_from_exception b bad_mode ENDPROC(el0_sync) -- cgit v1.2.3 From d99f5abc00687df8b32e3afaee52eac399fef69e Mon Sep 17 00:00:00 2001 From: Larry Bassel Date: Wed, 15 Oct 2014 16:09:39 -0700 Subject: arm64: enable context tracking Backport of the following patch to 3.14 LSK: commit 6c81fe7925cc4c42de49e17be21eb86d1173c3a7 Author: Larry Bassel Date: Fri May 30 12:34:15 2014 -0700 arm64: enable context tracking Make calls to ct_user_enter when the kernel is exited and ct_user_exit when the kernel is entered (in el0_da, el0_ia, el0_svc, el0_irq and all of the "error" paths). These macros expand to function calls which will only work properly if el0_sync and related code has been rearranged (in a previous patch of this series). The calls to ct_user_exit are made after hw debugging has been enabled (enable_dbg_and_irq). The call to ct_user_enter is made at the beginning of the kernel_exit macro. This patch is based on earlier work by Kevin Hilman. Save/restore optimizations were also done by Kevin. Acked-by: Will Deacon Reviewed-by: Kevin Hilman Tested-by: Kevin Hilman Signed-off-by: Larry Bassel Signed-off-by: Kevin Hilman Signed-off-by: Will Deacon Signed-off-by: Catalin Marinas Signed-off-by: Larry Bassel --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/thread_info.h | 5 ++++- arch/arm64/kernel/entry.S | 38 +++++++++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 43564fce31a8..372eb823e92f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -64,6 +64,7 @@ config ARM64 select RTC_LIB select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE + select HAVE_CONTEXT_TRACKING help ARM 64-bit (AArch64) Linux support. diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 0a8b2a97a32e..cebe0ff4c80e 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -103,6 +103,7 @@ static inline struct thread_info *current_thread_info(void) #define TIF_SIGPENDING 0 #define TIF_NEED_RESCHED 1 #define TIF_NOTIFY_RESUME 2 /* callback before returning to user */ +#define TIF_NOHZ 7 #define TIF_SYSCALL_TRACE 8 #define TIF_SYSCALL_AUDIT 9 #define TIF_SYSCALL_TRACEPOINT 10 @@ -118,6 +119,7 @@ static inline struct thread_info *current_thread_info(void) #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_NOHZ (1 << TIF_NOHZ) #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) @@ -128,7 +130,8 @@ static inline struct thread_info *current_thread_info(void) _TIF_NOTIFY_RESUME) #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ - _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) + _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \ + _TIF_NOHZ) #endif /* __KERNEL__ */ #endif /* __ASM_THREAD_INFO_H */ diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index c779ef357e43..e59b7b3500fc 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -29,6 +29,32 @@ #include #include +/* + * Context tracking subsystem. Used to instrument transitions + * between user and kernel mode. + */ + .macro ct_user_exit, syscall = 0 +#ifdef CONFIG_CONTEXT_TRACKING + bl context_tracking_user_exit + .if \syscall == 1 + /* + * Save/restore needed during syscalls. Restore syscall arguments from + * the values already saved on stack during kernel_entry. + */ + ldp x0, x1, [sp] + ldp x2, x3, [sp, #S_X2] + ldp x4, x5, [sp, #S_X4] + ldp x6, x7, [sp, #S_X6] + .endif +#endif + .endm + + .macro ct_user_enter +#ifdef CONFIG_CONTEXT_TRACKING + bl context_tracking_user_enter +#endif + .endm + /* * Bad Abort numbers *----------------- @@ -88,6 +114,7 @@ .macro kernel_exit, el, ret = 0 ldp x21, x22, [sp, #S_PC] // load ELR, SPSR .if \el == 0 + ct_user_enter ldr x23, [sp, #S_SP] // load return stack pointer .endif .if \ret @@ -424,6 +451,7 @@ el0_da: enable_dbg // enable interrupts before calling the main handler enable_irq + ct_user_exit bic x0, x26, #(0xff << 56) mov x1, x25 mov x2, sp @@ -439,6 +467,7 @@ el0_ia: enable_dbg // enable interrupts before calling the main handler enable_irq + ct_user_exit mov x0, x26 orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts mov x2, sp @@ -448,6 +477,7 @@ el0_fpsimd_acc: /* * Floating Point or Advanced SIMD access */ + ct_user_exit mov x0, x25 mov x1, sp adr lr, ret_from_exception @@ -456,6 +486,7 @@ el0_fpsimd_exc: /* * Floating Point or Advanced SIMD exception */ + ct_user_exit mov x0, x25 mov x1, sp adr lr, ret_from_exception @@ -479,9 +510,10 @@ el0_undef: /* * Undefined instruction */ - mov x0, sp // enable interrupts before calling the main handler + ct_user_exit enable_irq + mov x0, sp adr lr, ret_from_exception b do_undefinstr el0_dbg: @@ -489,12 +521,14 @@ el0_dbg: * Debug exception handling */ tbnz x24, #0, el0_inv // EL0 only + ct_user_exit mrs x0, far_el1 disable_step x1 mov x1, x25 mov x2, sp b do_debug_exception el0_inv: + ct_user_exit mov x0, sp mov x1, #BAD_SYNC mrs x2, esr_el1 @@ -513,6 +547,7 @@ el0_irq_naked: bl trace_hardirqs_off #endif + ct_user_exit irq_handler get_thread_info tsk @@ -635,6 +670,7 @@ el0_svc_naked: // compat entry point isb enable_dbg enable_irq + ct_user_exit 1 get_thread_info tsk ldr x16, [tsk, #TI_FLAGS] // check for syscall hooks -- cgit v1.2.3