From 0c91e7e07ebf08092bf8e28d8cd8d420732fc716 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 23 Apr 2013 16:45:40 -0400 Subject: ARM: cacheflush: add synchronization helpers for mixed cache state accesses Algorithms used by the MCPM layer rely on state variables which are accessed while the cache is either active or inactive, depending on the code path and the active state. This patch introduces generic cache maintenance helpers to provide the necessary cache synchronization for such state variables to always hit main memory in an ordered way. Signed-off-by: Nicolas Pitre Acked-by: Russell King Acked-by: Dave Martin --- arch/arm/include/asm/cacheflush.h | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) (limited to 'arch') diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index e1489c54cd1..bff71388e72 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h @@ -363,4 +363,79 @@ static inline void flush_cache_vunmap(unsigned long start, unsigned long end) flush_cache_all(); } +/* + * Memory synchronization helpers for mixed cached vs non cached accesses. + * + * Some synchronization algorithms have to set states in memory with the + * cache enabled or disabled depending on the code path. It is crucial + * to always ensure proper cache maintenance to update main memory right + * away in that case. + * + * Any cached write must be followed by a cache clean operation. + * Any cached read must be preceded by a cache invalidate operation. + * Yet, in the read case, a cache flush i.e. atomic clean+invalidate + * operation is needed to avoid discarding possible concurrent writes to the + * accessed memory. + * + * Also, in order to prevent a cached writer from interfering with an + * adjacent non-cached writer, each state variable must be located to + * a separate cache line. + */ + +/* + * This needs to be >= the max cache writeback size of all + * supported platforms included in the current kernel configuration. + * This is used to align state variables to their own cache lines. + */ +#define __CACHE_WRITEBACK_ORDER 6 /* guessed from existing platforms */ +#define __CACHE_WRITEBACK_GRANULE (1 << __CACHE_WRITEBACK_ORDER) + +/* + * There is no __cpuc_clean_dcache_area but we use it anyway for + * code intent clarity, and alias it to __cpuc_flush_dcache_area. + */ +#define __cpuc_clean_dcache_area __cpuc_flush_dcache_area + +/* + * Ensure preceding writes to *p by this CPU are visible to + * subsequent reads by other CPUs: + */ +static inline void __sync_cache_range_w(volatile void *p, size_t size) +{ + char *_p = (char *)p; + + __cpuc_clean_dcache_area(_p, size); + outer_clean_range(__pa(_p), __pa(_p + size)); +} + +/* + * Ensure preceding writes to *p by other CPUs are visible to + * subsequent reads by this CPU. We must be careful not to + * discard data simultaneously written by another CPU, hence the + * usage of flush rather than invalidate operations. + */ +static inline void __sync_cache_range_r(volatile void *p, size_t size) +{ + char *_p = (char *)p; + +#ifdef CONFIG_OUTER_CACHE + if (outer_cache.flush_range) { + /* + * Ensure dirty data migrated from other CPUs into our cache + * are cleaned out safely before the outer cache is cleaned: + */ + __cpuc_clean_dcache_area(_p, size); + + /* Clean and invalidate stale data for *p from outer ... */ + outer_flush_range(__pa(_p), __pa(_p + size)); + } +#endif + + /* ... and inner cache: */ + __cpuc_flush_dcache_area(_p, size); +} + +#define sync_cache_w(ptr) __sync_cache_range_w(ptr, sizeof *(ptr)) +#define sync_cache_r(ptr) __sync_cache_range_r(ptr, sizeof *(ptr)) + #endif -- cgit v1.2.3 From e8db288e05e588ad3f416b3a24354d60d02f35f2 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 12 Apr 2012 02:45:22 -0400 Subject: ARM: multi-cluster PM: secondary kernel entry code CPUs in cluster based systems, such as big.LITTLE, have special needs when entering the kernel due to a hotplug event, or when resuming from a deep sleep mode. This is vectorized so multiple CPUs can enter the kernel in parallel without serialization. The mcpm prefix stands for "multi cluster power management", however this is usable on single cluster systems as well. Only the basic structure is introduced here. This will be extended with later patches. In order not to complexify things more than they currently have to, the planned work to make runtime adjusted MPIDR based indexing and dynamic memory allocation for cluster states is postponed to a later cycle. The MAX_NR_CLUSTERS and MAX_CPUS_PER_CLUSTER static definitions should be sufficient for those systems expected to be available in the near future. Signed-off-by: Nicolas Pitre Reviewed-by: Santosh Shilimkar Reviewed-by: Will Deacon --- arch/arm/Kconfig | 8 +++++ arch/arm/common/Makefile | 1 + arch/arm/common/mcpm_entry.c | 22 ++++++++++++ arch/arm/common/mcpm_head.S | 86 ++++++++++++++++++++++++++++++++++++++++++++ arch/arm/include/asm/mcpm.h | 42 ++++++++++++++++++++++ 5 files changed, 159 insertions(+) create mode 100644 arch/arm/common/mcpm_entry.c create mode 100644 arch/arm/common/mcpm_head.S create mode 100644 arch/arm/include/asm/mcpm.h (limited to 'arch') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 2c3bdce1513..2d17275b5df 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1599,6 +1599,14 @@ config HAVE_ARM_TWD help This options enables support for the ARM timer and watchdog unit +config MCPM + bool "Multi-Cluster Power Management" + depends on CPU_V7 && SMP + help + This option provides the common power management infrastructure + for (multi-)cluster based systems, such as big.LITTLE based + systems. + choice prompt "Memory split" default VMSPLIT_3G diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile index dc8dd0de5c0..b070671033a 100644 --- a/arch/arm/common/Makefile +++ b/arch/arm/common/Makefile @@ -11,3 +11,4 @@ obj-$(CONFIG_SHARP_PARAM) += sharpsl_param.o obj-$(CONFIG_SHARP_SCOOP) += scoop.o obj-$(CONFIG_PCI_HOST_ITE8152) += it8152.o obj-$(CONFIG_ARM_TIMER_SP804) += timer-sp.o +obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o diff --git a/arch/arm/common/mcpm_entry.c b/arch/arm/common/mcpm_entry.c new file mode 100644 index 00000000000..7cbf70051ea --- /dev/null +++ b/arch/arm/common/mcpm_entry.c @@ -0,0 +1,22 @@ +/* + * arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM + * + * Created by: Nicolas Pitre, March 2012 + * Copyright: (C) 2012-2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER]; + +void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr) +{ + unsigned long val = ptr ? virt_to_phys(ptr) : 0; + mcpm_entry_vectors[cluster][cpu] = val; + sync_cache_w(&mcpm_entry_vectors[cluster][cpu]); +} diff --git a/arch/arm/common/mcpm_head.S b/arch/arm/common/mcpm_head.S new file mode 100644 index 00000000000..68c9903075a --- /dev/null +++ b/arch/arm/common/mcpm_head.S @@ -0,0 +1,86 @@ +/* + * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM + * + * Created by: Nicolas Pitre, March 2012 + * Copyright: (C) 2012-2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + + .macro pr_dbg string +#if defined(CONFIG_DEBUG_LL) && defined(DEBUG) + b 1901f +1902: .asciz "CPU" +1903: .asciz " cluster" +1904: .asciz ": \string" + .align +1901: adr r0, 1902b + bl printascii + mov r0, r9 + bl printhex8 + adr r0, 1903b + bl printascii + mov r0, r10 + bl printhex8 + adr r0, 1904b + bl printascii +#endif + .endm + + .arm + .align + +ENTRY(mcpm_entry_point) + + THUMB( adr r12, BSYM(1f) ) + THUMB( bx r12 ) + THUMB( .thumb ) +1: + mrc p15, 0, r0, c0, c0, 5 @ MPIDR + ubfx r9, r0, #0, #8 @ r9 = cpu + ubfx r10, r0, #8, #8 @ r10 = cluster + mov r3, #MAX_CPUS_PER_CLUSTER + mla r4, r3, r10, r9 @ r4 = canonical CPU index + cmp r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS) + blo 2f + + /* We didn't expect this CPU. Try to cheaply make it quiet. */ +1: wfi + wfe + b 1b + +2: pr_dbg "kernel mcpm_entry_point\n" + + /* + * MMU is off so we need to get to mcpm_entry_vectors in a + * position independent way. + */ + adr r5, 3f + ldr r6, [r5] + add r6, r5, r6 @ r6 = mcpm_entry_vectors + +mcpm_entry_gated: + ldr r5, [r6, r4, lsl #2] @ r5 = CPU entry vector + cmp r5, #0 + wfeeq + beq mcpm_entry_gated + pr_dbg "released\n" + bx r5 + + .align 2 + +3: .word mcpm_entry_vectors - . + +ENDPROC(mcpm_entry_point) + + .bss + .align 5 + + .type mcpm_entry_vectors, #object +ENTRY(mcpm_entry_vectors) + .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER diff --git a/arch/arm/include/asm/mcpm.h b/arch/arm/include/asm/mcpm.h new file mode 100644 index 00000000000..470a417d135 --- /dev/null +++ b/arch/arm/include/asm/mcpm.h @@ -0,0 +1,42 @@ +/* + * arch/arm/include/asm/mcpm.h + * + * Created by: Nicolas Pitre, April 2012 + * Copyright: (C) 2012-2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef MCPM_H +#define MCPM_H + +/* + * Maximum number of possible clusters / CPUs per cluster. + * + * This should be sufficient for quite a while, while keeping the + * (assembly) code simpler. When this starts to grow then we'll have + * to consider dynamic allocation. + */ +#define MAX_CPUS_PER_CLUSTER 4 +#define MAX_NR_CLUSTERS 2 + +#ifndef __ASSEMBLY__ + +/* + * Platform specific code should use this symbol to set up secondary + * entry location for processors to use when released from reset. + */ +extern void mcpm_entry_point(void); + +/* + * This is used to indicate where the given CPU from given cluster should + * branch once it is ready to re-enter the kernel using ptr, or NULL if it + * should be gated. A gated CPU is held in a WFE loop until its vector + * becomes non NULL. + */ +void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr); + +#endif /* ! __ASSEMBLY__ */ +#endif -- cgit v1.2.3 From 7c2b860534d02d11923dd0504b961f21508173f1 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 20 Sep 2012 16:05:37 -0400 Subject: ARM: mcpm: introduce the CPU/cluster power API This is the basic API used to handle the powering up/down of individual CPUs in a (multi-)cluster system. The platform specific backend implementation has the responsibility to also handle the cluster level power as well when the first/last CPU in a cluster is brought up/down. Signed-off-by: Nicolas Pitre Reviewed-by: Santosh Shilimkar Reviewed-by: Will Deacon --- arch/arm/common/mcpm_entry.c | 91 +++++++++++++++++++++++++++++++++++++++++++ arch/arm/include/asm/mcpm.h | 92 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) (limited to 'arch') diff --git a/arch/arm/common/mcpm_entry.c b/arch/arm/common/mcpm_entry.c index 7cbf70051ea..5d72889a58a 100644 --- a/arch/arm/common/mcpm_entry.c +++ b/arch/arm/common/mcpm_entry.c @@ -9,8 +9,13 @@ * published by the Free Software Foundation. */ +#include +#include +#include + #include #include +#include extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER]; @@ -20,3 +25,89 @@ void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr) mcpm_entry_vectors[cluster][cpu] = val; sync_cache_w(&mcpm_entry_vectors[cluster][cpu]); } + +static const struct mcpm_platform_ops *platform_ops; + +int __init mcpm_platform_register(const struct mcpm_platform_ops *ops) +{ + if (platform_ops) + return -EBUSY; + platform_ops = ops; + return 0; +} + +int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster) +{ + if (!platform_ops) + return -EUNATCH; /* try not to shadow power_up errors */ + might_sleep(); + return platform_ops->power_up(cpu, cluster); +} + +typedef void (*phys_reset_t)(unsigned long); + +void mcpm_cpu_power_down(void) +{ + phys_reset_t phys_reset; + + BUG_ON(!platform_ops); + BUG_ON(!irqs_disabled()); + + /* + * Do this before calling into the power_down method, + * as it might not always be safe to do afterwards. + */ + setup_mm_for_reboot(); + + platform_ops->power_down(); + + /* + * It is possible for a power_up request to happen concurrently + * with a power_down request for the same CPU. In this case the + * power_down method might not be able to actually enter a + * powered down state with the WFI instruction if the power_up + * method has removed the required reset condition. The + * power_down method is then allowed to return. We must perform + * a re-entry in the kernel as if the power_up method just had + * deasserted reset on the CPU. + * + * To simplify race issues, the platform specific implementation + * must accommodate for the possibility of unordered calls to + * power_down and power_up with a usage count. Therefore, if a + * call to power_up is issued for a CPU that is not down, then + * the next call to power_down must not attempt a full shutdown + * but only do the minimum (normally disabling L1 cache and CPU + * coherency) and return just as if a concurrent power_up request + * had happened as described above. + */ + + phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset); + phys_reset(virt_to_phys(mcpm_entry_point)); + + /* should never get here */ + BUG(); +} + +void mcpm_cpu_suspend(u64 expected_residency) +{ + phys_reset_t phys_reset; + + BUG_ON(!platform_ops); + BUG_ON(!irqs_disabled()); + + /* Very similar to mcpm_cpu_power_down() */ + setup_mm_for_reboot(); + platform_ops->suspend(expected_residency); + phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset); + phys_reset(virt_to_phys(mcpm_entry_point)); + BUG(); +} + +int mcpm_cpu_powered_up(void) +{ + if (!platform_ops) + return -EUNATCH; + if (platform_ops->powered_up) + platform_ops->powered_up(); + return 0; +} diff --git a/arch/arm/include/asm/mcpm.h b/arch/arm/include/asm/mcpm.h index 470a417d135..627761fce78 100644 --- a/arch/arm/include/asm/mcpm.h +++ b/arch/arm/include/asm/mcpm.h @@ -38,5 +38,97 @@ extern void mcpm_entry_point(void); */ void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr); +/* + * CPU/cluster power operations API for higher subsystems to use. + */ + +/** + * mcpm_cpu_power_up - make given CPU in given cluster runable + * + * @cpu: CPU number within given cluster + * @cluster: cluster number for the CPU + * + * The identified CPU is brought out of reset. If the cluster was powered + * down then it is brought up as well, taking care not to let the other CPUs + * in the cluster run, and ensuring appropriate cluster setup. + * + * Caller must ensure the appropriate entry vector is initialized with + * mcpm_set_entry_vector() prior to calling this. + * + * This must be called in a sleepable context. However, the implementation + * is strongly encouraged to return early and let the operation happen + * asynchronously, especially when significant delays are expected. + * + * If the operation cannot be performed then an error code is returned. + */ +int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster); + +/** + * mcpm_cpu_power_down - power the calling CPU down + * + * The calling CPU is powered down. + * + * If this CPU is found to be the "last man standing" in the cluster + * then the cluster is prepared for power-down too. + * + * This must be called with interrupts disabled. + * + * This does not return. Re-entry in the kernel is expected via + * mcpm_entry_point. + */ +void mcpm_cpu_power_down(void); + +/** + * mcpm_cpu_suspend - bring the calling CPU in a suspended state + * + * @expected_residency: duration in microseconds the CPU is expected + * to remain suspended, or 0 if unknown/infinity. + * + * The calling CPU is suspended. The expected residency argument is used + * as a hint by the platform specific backend to implement the appropriate + * sleep state level according to the knowledge it has on wake-up latency + * for the given hardware. + * + * If this CPU is found to be the "last man standing" in the cluster + * then the cluster may be prepared for power-down too, if the expected + * residency makes it worthwhile. + * + * This must be called with interrupts disabled. + * + * This does not return. Re-entry in the kernel is expected via + * mcpm_entry_point. + */ +void mcpm_cpu_suspend(u64 expected_residency); + +/** + * mcpm_cpu_powered_up - housekeeping workafter a CPU has been powered up + * + * This lets the platform specific backend code perform needed housekeeping + * work. This must be called by the newly activated CPU as soon as it is + * fully operational in kernel space, before it enables interrupts. + * + * If the operation cannot be performed then an error code is returned. + */ +int mcpm_cpu_powered_up(void); + +/* + * Platform specific methods used in the implementation of the above API. + */ +struct mcpm_platform_ops { + int (*power_up)(unsigned int cpu, unsigned int cluster); + void (*power_down)(void); + void (*suspend)(u64); + void (*powered_up)(void); +}; + +/** + * mcpm_platform_register - register platform specific power methods + * + * @ops: mcpm_platform_ops structure to register + * + * An error is returned if the registration has been done previously. + */ +int __init mcpm_platform_register(const struct mcpm_platform_ops *ops); + #endif /* ! __ASSEMBLY__ */ #endif -- cgit v1.2.3 From 7fe31d28e839f9565c8176ec584676a045970802 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Tue, 17 Jul 2012 14:25:42 +0100 Subject: ARM: mcpm: introduce helpers for platform coherency exit/setup This provides helper methods to coordinate between CPUs coming down and CPUs going up, as well as documentation on the used algorithms, so that cluster teardown and setup operations are not done for a cluster simultaneously. For use in the power_down() implementation: * __mcpm_cpu_going_down(unsigned int cluster, unsigned int cpu) * __mcpm_outbound_enter_critical(unsigned int cluster) * __mcpm_outbound_leave_critical(unsigned int cluster) * __mcpm_cpu_down(unsigned int cluster, unsigned int cpu) The power_up_setup() helper should do platform-specific setup in preparation for turning the CPU on, such as invalidating local caches or entering coherency. It must be assembler for now, since it must run before the MMU can be switched on. It is passed the affinity level for which initialization should be performed. Because the mcpm_sync_struct content is looked-up and modified with the cache enabled or disabled depending on the code path, it is crucial to always ensure proper cache maintenance to update main memory right away. The sync_cache_*() helpers are used to that end. Also, in order to prevent a cached writer from interfering with an adjacent non-cached writer, we ensure each state variable is located to a separate cache line. Thanks to Nicolas Pitre and Achin Gupta for the help with this patch. Signed-off-by: Dave Martin Signed-off-by: Nicolas Pitre Reviewed-by: Will Deacon --- arch/arm/common/mcpm_entry.c | 150 ++++++++++++++++++++++++++++++++++++++++++ arch/arm/common/mcpm_head.S | 106 ++++++++++++++++++++++++++++- arch/arm/include/asm/mcpm.h | 73 ++++++++++++++++++++ arch/arm/kernel/asm-offsets.c | 3 + 4 files changed, 330 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arm/common/mcpm_entry.c b/arch/arm/common/mcpm_entry.c index 5d72889a58a..370236dd1a0 100644 --- a/arch/arm/common/mcpm_entry.c +++ b/arch/arm/common/mcpm_entry.c @@ -16,6 +16,7 @@ #include #include #include +#include extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER]; @@ -111,3 +112,152 @@ int mcpm_cpu_powered_up(void) platform_ops->powered_up(); return 0; } + +struct sync_struct mcpm_sync; + +/* + * __mcpm_cpu_going_down: Indicates that the cpu is being torn down. + * This must be called at the point of committing to teardown of a CPU. + * The CPU cache (SCTRL.C bit) is expected to still be active. + */ +void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster) +{ + mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN; + sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu); +} + +/* + * __mcpm_cpu_down: Indicates that cpu teardown is complete and that the + * cluster can be torn down without disrupting this CPU. + * To avoid deadlocks, this must be called before a CPU is powered down. + * The CPU cache (SCTRL.C bit) is expected to be off. + * However L2 cache might or might not be active. + */ +void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster) +{ + dmb(); + mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN; + sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu); + dsb_sev(); +} + +/* + * __mcpm_outbound_leave_critical: Leave the cluster teardown critical section. + * @state: the final state of the cluster: + * CLUSTER_UP: no destructive teardown was done and the cluster has been + * restored to the previous state (CPU cache still active); or + * CLUSTER_DOWN: the cluster has been torn-down, ready for power-off + * (CPU cache disabled, L2 cache either enabled or disabled). + */ +void __mcpm_outbound_leave_critical(unsigned int cluster, int state) +{ + dmb(); + mcpm_sync.clusters[cluster].cluster = state; + sync_cache_w(&mcpm_sync.clusters[cluster].cluster); + dsb_sev(); +} + +/* + * __mcpm_outbound_enter_critical: Enter the cluster teardown critical section. + * This function should be called by the last man, after local CPU teardown + * is complete. CPU cache expected to be active. + * + * Returns: + * false: the critical section was not entered because an inbound CPU was + * observed, or the cluster is already being set up; + * true: the critical section was entered: it is now safe to tear down the + * cluster. + */ +bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster) +{ + unsigned int i; + struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster]; + + /* Warn inbound CPUs that the cluster is being torn down: */ + c->cluster = CLUSTER_GOING_DOWN; + sync_cache_w(&c->cluster); + + /* Back out if the inbound cluster is already in the critical region: */ + sync_cache_r(&c->inbound); + if (c->inbound == INBOUND_COMING_UP) + goto abort; + + /* + * Wait for all CPUs to get out of the GOING_DOWN state, so that local + * teardown is complete on each CPU before tearing down the cluster. + * + * If any CPU has been woken up again from the DOWN state, then we + * shouldn't be taking the cluster down at all: abort in that case. + */ + sync_cache_r(&c->cpus); + for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) { + int cpustate; + + if (i == cpu) + continue; + + while (1) { + cpustate = c->cpus[i].cpu; + if (cpustate != CPU_GOING_DOWN) + break; + + wfe(); + sync_cache_r(&c->cpus[i].cpu); + } + + switch (cpustate) { + case CPU_DOWN: + continue; + + default: + goto abort; + } + } + + return true; + +abort: + __mcpm_outbound_leave_critical(cluster, CLUSTER_UP); + return false; +} + +int __mcpm_cluster_state(unsigned int cluster) +{ + sync_cache_r(&mcpm_sync.clusters[cluster].cluster); + return mcpm_sync.clusters[cluster].cluster; +} + +extern unsigned long mcpm_power_up_setup_phys; + +int __init mcpm_sync_init( + void (*power_up_setup)(unsigned int affinity_level)) +{ + unsigned int i, j, mpidr, this_cluster; + + BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync); + BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1)); + + /* + * Set initial CPU and cluster states. + * Only one cluster is assumed to be active at this point. + */ + for (i = 0; i < MAX_NR_CLUSTERS; i++) { + mcpm_sync.clusters[i].cluster = CLUSTER_DOWN; + mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP; + for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++) + mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN; + } + mpidr = read_cpuid_mpidr(); + this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); + for_each_online_cpu(i) + mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP; + mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP; + sync_cache_w(&mcpm_sync); + + if (power_up_setup) { + mcpm_power_up_setup_phys = virt_to_phys(power_up_setup); + sync_cache_w(&mcpm_power_up_setup_phys); + } + + return 0; +} diff --git a/arch/arm/common/mcpm_head.S b/arch/arm/common/mcpm_head.S index 68c9903075a..7d729bd7267 100644 --- a/arch/arm/common/mcpm_head.S +++ b/arch/arm/common/mcpm_head.S @@ -7,11 +7,19 @@ * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. + * + * + * Refer to Documentation/arm/cluster-pm-race-avoidance.txt + * for details of the synchronisation algorithms used here. */ #include #include +.if MCPM_SYNC_CLUSTER_CPUS +.error "cpus must be the first member of struct mcpm_sync_struct" +.endif + .macro pr_dbg string #if defined(CONFIG_DEBUG_LL) && defined(DEBUG) b 1901f @@ -57,24 +65,114 @@ ENTRY(mcpm_entry_point) 2: pr_dbg "kernel mcpm_entry_point\n" /* - * MMU is off so we need to get to mcpm_entry_vectors in a + * MMU is off so we need to get to various variables in a * position independent way. */ adr r5, 3f - ldr r6, [r5] + ldmia r5, {r6, r7, r8} add r6, r5, r6 @ r6 = mcpm_entry_vectors + ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys + add r8, r5, r8 @ r8 = mcpm_sync + + mov r0, #MCPM_SYNC_CLUSTER_SIZE + mla r8, r0, r10, r8 @ r8 = sync cluster base + + @ Signal that this CPU is coming UP: + mov r0, #CPU_COMING_UP + mov r5, #MCPM_SYNC_CPU_SIZE + mla r5, r9, r5, r8 @ r5 = sync cpu address + strb r0, [r5] + + @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN + @ state, because there is at least one active CPU (this CPU). + + @ Note: the following is racy as another CPU might be testing + @ the same flag at the same moment. That'll be fixed later. + ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] + cmp r0, #CLUSTER_UP @ cluster already up? + bne mcpm_setup @ if not, set up the cluster + + @ Otherwise, skip setup: + b mcpm_setup_complete + +mcpm_setup: + @ Control dependency implies strb not observable before previous ldrb. + + @ Signal that the cluster is being brought up: + mov r0, #INBOUND_COMING_UP + strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] + dmb + + @ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this + @ point onwards will observe INBOUND_COMING_UP and abort. + + @ Wait for any previously-pending cluster teardown operations to abort + @ or complete: +mcpm_teardown_wait: + ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] + cmp r0, #CLUSTER_GOING_DOWN + bne first_man_setup + wfe + b mcpm_teardown_wait + +first_man_setup: + dmb + + @ If the outbound gave up before teardown started, skip cluster setup: + + cmp r0, #CLUSTER_UP + beq mcpm_setup_leave + + @ power_up_setup is now responsible for setting up the cluster: + + cmp r7, #0 + mov r0, #1 @ second (cluster) affinity level + blxne r7 @ Call power_up_setup if defined + dmb + + mov r0, #CLUSTER_UP + strb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] + dmb + +mcpm_setup_leave: + @ Leave the cluster setup critical section: + + mov r0, #INBOUND_NOT_COMING_UP + strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] + dsb + sev + +mcpm_setup_complete: + @ If a platform-specific CPU setup hook is needed, it is + @ called from here. + + cmp r7, #0 + mov r0, #0 @ first (CPU) affinity level + blxne r7 @ Call power_up_setup if defined + dmb + + @ Mark the CPU as up: + + mov r0, #CPU_UP + strb r0, [r5] + + @ Observability order of CPU_UP and opening of the gate does not matter. mcpm_entry_gated: ldr r5, [r6, r4, lsl #2] @ r5 = CPU entry vector cmp r5, #0 wfeeq beq mcpm_entry_gated + dmb + pr_dbg "released\n" bx r5 .align 2 3: .word mcpm_entry_vectors - . + .word mcpm_power_up_setup_phys - 3b + .word mcpm_sync - 3b ENDPROC(mcpm_entry_point) @@ -84,3 +182,7 @@ ENDPROC(mcpm_entry_point) .type mcpm_entry_vectors, #object ENTRY(mcpm_entry_vectors) .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER + + .type mcpm_power_up_setup_phys, #object +ENTRY(mcpm_power_up_setup_phys) + .space 4 @ set by mcpm_sync_init() diff --git a/arch/arm/include/asm/mcpm.h b/arch/arm/include/asm/mcpm.h index 627761fce78..3046e90210c 100644 --- a/arch/arm/include/asm/mcpm.h +++ b/arch/arm/include/asm/mcpm.h @@ -24,6 +24,9 @@ #ifndef __ASSEMBLY__ +#include +#include + /* * Platform specific code should use this symbol to set up secondary * entry location for processors to use when released from reset. @@ -130,5 +133,75 @@ struct mcpm_platform_ops { */ int __init mcpm_platform_register(const struct mcpm_platform_ops *ops); +/* Synchronisation structures for coordinating safe cluster setup/teardown: */ + +/* + * When modifying this structure, make sure you update the MCPM_SYNC_ defines + * to match. + */ +struct mcpm_sync_struct { + /* individual CPU states */ + struct { + s8 cpu __aligned(__CACHE_WRITEBACK_GRANULE); + } cpus[MAX_CPUS_PER_CLUSTER]; + + /* cluster state */ + s8 cluster __aligned(__CACHE_WRITEBACK_GRANULE); + + /* inbound-side state */ + s8 inbound __aligned(__CACHE_WRITEBACK_GRANULE); +}; + +struct sync_struct { + struct mcpm_sync_struct clusters[MAX_NR_CLUSTERS]; +}; + +extern unsigned long sync_phys; /* physical address of *mcpm_sync */ + +void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster); +void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster); +void __mcpm_outbound_leave_critical(unsigned int cluster, int state); +bool __mcpm_outbound_enter_critical(unsigned int this_cpu, unsigned int cluster); +int __mcpm_cluster_state(unsigned int cluster); + +int __init mcpm_sync_init( + void (*power_up_setup)(unsigned int affinity_level)); + +#else + +/* + * asm-offsets.h causes trouble when included in .c files, and cacheflush.h + * cannot be included in asm files. Let's work around the conflict like this. + */ +#include +#define __CACHE_WRITEBACK_GRANULE CACHE_WRITEBACK_GRANULE + #endif /* ! __ASSEMBLY__ */ + +/* Definitions for mcpm_sync_struct */ +#define CPU_DOWN 0x11 +#define CPU_COMING_UP 0x12 +#define CPU_UP 0x13 +#define CPU_GOING_DOWN 0x14 + +#define CLUSTER_DOWN 0x21 +#define CLUSTER_UP 0x22 +#define CLUSTER_GOING_DOWN 0x23 + +#define INBOUND_NOT_COMING_UP 0x31 +#define INBOUND_COMING_UP 0x32 + +/* + * Offsets for the mcpm_sync_struct members, for use in asm. + * We don't want to make them global to the kernel via asm-offsets.c. + */ +#define MCPM_SYNC_CLUSTER_CPUS 0 +#define MCPM_SYNC_CPU_SIZE __CACHE_WRITEBACK_GRANULE +#define MCPM_SYNC_CLUSTER_CLUSTER \ + (MCPM_SYNC_CLUSTER_CPUS + MCPM_SYNC_CPU_SIZE * MAX_CPUS_PER_CLUSTER) +#define MCPM_SYNC_CLUSTER_INBOUND \ + (MCPM_SYNC_CLUSTER_CLUSTER + __CACHE_WRITEBACK_GRANULE) +#define MCPM_SYNC_CLUSTER_SIZE \ + (MCPM_SYNC_CLUSTER_INBOUND + __CACHE_WRITEBACK_GRANULE) + #endif diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 923eec7105c..1bed82a0a9e 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -149,6 +149,9 @@ int main(void) DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL); DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE); DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE); + BLANK(); + DEFINE(CACHE_WRITEBACK_GRANULE, __CACHE_WRITEBACK_GRANULE); + BLANK(); #ifdef CONFIG_KVM_ARM_HOST DEFINE(VCPU_KVM, offsetof(struct kvm_vcpu, kvm)); DEFINE(VCPU_MIDR, offsetof(struct kvm_vcpu, arch.midr)); -- cgit v1.2.3 From 9762f12d3e05c8d6a0651f4c7e76ae72ce27fc3a Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Fri, 17 Aug 2012 16:07:01 +0100 Subject: ARM: mcpm: Add baremetal voting mutexes This patch adds a simple low-level voting mutex implementation to be used to arbitrate during first man selection when no load/store exclusive instructions are usable. For want of a better name, these are called "vlocks". (I was tempted to call them ballot locks, but "block" is way too confusing an abbreviation...) There is no function to wait for the lock to be released, and no vlock_lock() function since we don't need these at the moment. These could straightforwardly be added if vlocks get used for other purposes. For architectural correctness even Strongly-Ordered memory accesses require barriers in order to guarantee that multiple CPUs have a coherent view of the ordering of memory accesses. Whether or not this matters depends on hardware implementation details of the memory system. Since the purpose of this code is to provide a clean, generic locking mechanism with no platform-specific dependencies the barriers should be present to avoid unpleasant surprises on future platforms. Note: * When taking the lock, we don't care about implicit background memory operations and other signalling which may be pending, because those are not part of the critical section anyway. A DMB is sufficient to ensure correctly observed ordering if the explicit memory accesses in vlock_trylock. * No barrier is required after checking the election result, because the result is determined by the store to VLOCK_OWNER_OFFSET and is already globally observed due to the barriers in voting_end. This means that global agreement on the winner is guaranteed, even before the winner is known locally. Signed-off-by: Dave Martin Signed-off-by: Nicolas Pitre Reviewed-by: Santosh Shilimkar Reviewed-by: Will Deacon --- arch/arm/common/vlock.S | 108 ++++++++++++++++++++++++++++++++++++++++++++++++ arch/arm/common/vlock.h | 29 +++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 arch/arm/common/vlock.S create mode 100644 arch/arm/common/vlock.h (limited to 'arch') diff --git a/arch/arm/common/vlock.S b/arch/arm/common/vlock.S new file mode 100644 index 00000000000..ff198583f68 --- /dev/null +++ b/arch/arm/common/vlock.S @@ -0,0 +1,108 @@ +/* + * vlock.S - simple voting lock implementation for ARM + * + * Created by: Dave Martin, 2012-08-16 + * Copyright: (C) 2012-2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * + * This algorithm is described in more detail in + * Documentation/arm/vlocks.txt. + */ + +#include +#include "vlock.h" + +/* Select different code if voting flags can fit in a single word. */ +#if VLOCK_VOTING_SIZE > 4 +#define FEW(x...) +#define MANY(x...) x +#else +#define FEW(x...) x +#define MANY(x...) +#endif + +@ voting lock for first-man coordination + +.macro voting_begin rbase:req, rcpu:req, rscratch:req + mov \rscratch, #1 + strb \rscratch, [\rbase, \rcpu] + dmb +.endm + +.macro voting_end rbase:req, rcpu:req, rscratch:req + dmb + mov \rscratch, #0 + strb \rscratch, [\rbase, \rcpu] + dsb + sev +.endm + +/* + * The vlock structure must reside in Strongly-Ordered or Device memory. + * This implementation deliberately eliminates most of the barriers which + * would be required for other memory types, and assumes that independent + * writes to neighbouring locations within a cacheline do not interfere + * with one another. + */ + +@ r0: lock structure base +@ r1: CPU ID (0-based index within cluster) +ENTRY(vlock_trylock) + add r1, r1, #VLOCK_VOTING_OFFSET + + voting_begin r0, r1, r2 + + ldrb r2, [r0, #VLOCK_OWNER_OFFSET] @ check whether lock is held + cmp r2, #VLOCK_OWNER_NONE + bne trylock_fail @ fail if so + + @ Control dependency implies strb not observable before previous ldrb. + + strb r1, [r0, #VLOCK_OWNER_OFFSET] @ submit my vote + + voting_end r0, r1, r2 @ implies DMB + + @ Wait for the current round of voting to finish: + + MANY( mov r3, #VLOCK_VOTING_OFFSET ) +0: + MANY( ldr r2, [r0, r3] ) + FEW( ldr r2, [r0, #VLOCK_VOTING_OFFSET] ) + cmp r2, #0 + wfene + bne 0b + MANY( add r3, r3, #4 ) + MANY( cmp r3, #VLOCK_VOTING_OFFSET + VLOCK_VOTING_SIZE ) + MANY( bne 0b ) + + @ Check who won: + + dmb + ldrb r2, [r0, #VLOCK_OWNER_OFFSET] + eor r0, r1, r2 @ zero if I won, else nonzero + bx lr + +trylock_fail: + voting_end r0, r1, r2 + mov r0, #1 @ nonzero indicates that I lost + bx lr +ENDPROC(vlock_trylock) + +@ r0: lock structure base +ENTRY(vlock_unlock) + dmb + mov r1, #VLOCK_OWNER_NONE + strb r1, [r0, #VLOCK_OWNER_OFFSET] + dsb + sev + bx lr +ENDPROC(vlock_unlock) diff --git a/arch/arm/common/vlock.h b/arch/arm/common/vlock.h new file mode 100644 index 00000000000..3b441475a59 --- /dev/null +++ b/arch/arm/common/vlock.h @@ -0,0 +1,29 @@ +/* + * vlock.h - simple voting lock implementation + * + * Created by: Dave Martin, 2012-08-16 + * Copyright: (C) 2012-2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __VLOCK_H +#define __VLOCK_H + +#include + +/* Offsets and sizes are rounded to a word (4 bytes) */ +#define VLOCK_OWNER_OFFSET 0 +#define VLOCK_VOTING_OFFSET 4 +#define VLOCK_VOTING_SIZE ((MAX_CPUS_PER_CLUSTER + 3) / 4 * 4) +#define VLOCK_SIZE (VLOCK_VOTING_OFFSET + VLOCK_VOTING_SIZE) +#define VLOCK_OWNER_NONE 0 + +#endif /* ! __VLOCK_H */ -- cgit v1.2.3 From 1ae98561b16f305e43151405f226727c00ee52bc Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Fri, 17 Aug 2012 16:07:02 +0100 Subject: ARM: mcpm_head.S: vlock-based first man election Instead of requiring the first man to be elected in advance (which can be suboptimal in some situations), this patch uses a per- cluster mutex to co-ordinate selection of the first man. This should also make it more feasible to reuse this code path for asynchronous cluster resume (as in CPUidle scenarios). We must ensure that the vlock data doesn't share a cacheline with anything else, or dirty cache eviction could corrupt it. Signed-off-by: Dave Martin Signed-off-by: Nicolas Pitre Reviewed-by: Santosh Shilimkar Reviewed-by: Will Deacon --- arch/arm/common/Makefile | 2 +- arch/arm/common/mcpm_head.S | 41 ++++++++++++++++++++++++++++++++++++----- arch/arm/kernel/asm-offsets.c | 1 + 3 files changed, 38 insertions(+), 6 deletions(-) (limited to 'arch') diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile index b070671033a..9ec273188cc 100644 --- a/arch/arm/common/Makefile +++ b/arch/arm/common/Makefile @@ -11,4 +11,4 @@ obj-$(CONFIG_SHARP_PARAM) += sharpsl_param.o obj-$(CONFIG_SHARP_SCOOP) += scoop.o obj-$(CONFIG_PCI_HOST_ITE8152) += it8152.o obj-$(CONFIG_ARM_TIMER_SP804) += timer-sp.o -obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o +obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o vlock.o diff --git a/arch/arm/common/mcpm_head.S b/arch/arm/common/mcpm_head.S index 7d729bd7267..8178705c4b2 100644 --- a/arch/arm/common/mcpm_head.S +++ b/arch/arm/common/mcpm_head.S @@ -16,6 +16,8 @@ #include #include +#include "vlock.h" + .if MCPM_SYNC_CLUSTER_CPUS .error "cpus must be the first member of struct mcpm_sync_struct" .endif @@ -69,10 +71,11 @@ ENTRY(mcpm_entry_point) * position independent way. */ adr r5, 3f - ldmia r5, {r6, r7, r8} + ldmia r5, {r6, r7, r8, r11} add r6, r5, r6 @ r6 = mcpm_entry_vectors ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys add r8, r5, r8 @ r8 = mcpm_sync + add r11, r5, r11 @ r11 = first_man_locks mov r0, #MCPM_SYNC_CLUSTER_SIZE mla r8, r0, r10, r8 @ r8 = sync cluster base @@ -86,13 +89,22 @@ ENTRY(mcpm_entry_point) @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN @ state, because there is at least one active CPU (this CPU). - @ Note: the following is racy as another CPU might be testing - @ the same flag at the same moment. That'll be fixed later. + mov r0, #VLOCK_SIZE + mla r11, r0, r10, r11 @ r11 = cluster first man lock + mov r0, r11 + mov r1, r9 @ cpu + bl vlock_trylock @ implies DMB + + cmp r0, #0 @ failed to get the lock? + bne mcpm_setup_wait @ wait for cluster setup if so + ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] cmp r0, #CLUSTER_UP @ cluster already up? bne mcpm_setup @ if not, set up the cluster - @ Otherwise, skip setup: + @ Otherwise, release the first man lock and skip setup: + mov r0, r11 + bl vlock_unlock b mcpm_setup_complete mcpm_setup: @@ -142,6 +154,19 @@ mcpm_setup_leave: dsb sev + mov r0, r11 + bl vlock_unlock @ implies DMB + b mcpm_setup_complete + + @ In the contended case, non-first men wait here for cluster setup + @ to complete: +mcpm_setup_wait: + ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] + cmp r0, #CLUSTER_UP + wfene + bne mcpm_setup_wait + dmb + mcpm_setup_complete: @ If a platform-specific CPU setup hook is needed, it is @ called from here. @@ -173,11 +198,17 @@ mcpm_entry_gated: 3: .word mcpm_entry_vectors - . .word mcpm_power_up_setup_phys - 3b .word mcpm_sync - 3b + .word first_man_locks - 3b ENDPROC(mcpm_entry_point) .bss - .align 5 + + .align CACHE_WRITEBACK_ORDER + .type first_man_locks, #object +first_man_locks: + .space VLOCK_SIZE * MAX_NR_CLUSTERS + .align CACHE_WRITEBACK_ORDER .type mcpm_entry_vectors, #object ENTRY(mcpm_entry_vectors) diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 1bed82a0a9e..3f088225e71 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -150,6 +150,7 @@ int main(void) DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE); DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE); BLANK(); + DEFINE(CACHE_WRITEBACK_ORDER, __CACHE_WRITEBACK_ORDER); DEFINE(CACHE_WRITEBACK_GRANULE, __CACHE_WRITEBACK_GRANULE); BLANK(); #ifdef CONFIG_KVM_ARM_HOST -- cgit v1.2.3 From 9ff221bad8869f73141c6a3c187afe2e933c991f Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Mon, 19 Nov 2012 16:02:17 -0500 Subject: ARM: mcpm: generic SMP secondary bringup and hotplug support Now that the cluster power API is in place, we can use it for SMP secondary bringup and CPU hotplug in a generic fashion. Signed-off-by: Nicolas Pitre Reviewed-by: Santosh Shilimkar Reviewed-by: Will Deacon --- arch/arm/common/Makefile | 2 +- arch/arm/common/mcpm_platsmp.c | 87 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 arch/arm/common/mcpm_platsmp.c (limited to 'arch') diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile index 9ec273188cc..48c0ee57817 100644 --- a/arch/arm/common/Makefile +++ b/arch/arm/common/Makefile @@ -11,4 +11,4 @@ obj-$(CONFIG_SHARP_PARAM) += sharpsl_param.o obj-$(CONFIG_SHARP_SCOOP) += scoop.o obj-$(CONFIG_PCI_HOST_ITE8152) += it8152.o obj-$(CONFIG_ARM_TIMER_SP804) += timer-sp.o -obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o vlock.o +obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o mcpm_platsmp.o vlock.o diff --git a/arch/arm/common/mcpm_platsmp.c b/arch/arm/common/mcpm_platsmp.c new file mode 100644 index 00000000000..e7e3235b8b6 --- /dev/null +++ b/arch/arm/common/mcpm_platsmp.c @@ -0,0 +1,87 @@ +/* + * linux/arch/arm/mach-vexpress/mcpm_platsmp.c + * + * Created by: Nicolas Pitre, November 2012 + * Copyright: (C) 2012-2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Code to handle secondary CPU bringup and hotplug for the cluster power API. + */ + +#include +#include +#include + +#include + +#include +#include +#include + +static void __init simple_smp_init_cpus(void) +{ +} + +static int __cpuinit mcpm_boot_secondary(unsigned int cpu, struct task_struct *idle) +{ + unsigned int mpidr, pcpu, pcluster, ret; + extern void secondary_startup(void); + + mpidr = cpu_logical_map(cpu); + pcpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); + pcluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); + pr_debug("%s: logical CPU %d is physical CPU %d cluster %d\n", + __func__, cpu, pcpu, pcluster); + + mcpm_set_entry_vector(pcpu, pcluster, NULL); + ret = mcpm_cpu_power_up(pcpu, pcluster); + if (ret) + return ret; + mcpm_set_entry_vector(pcpu, pcluster, secondary_startup); + arch_send_wakeup_ipi_mask(cpumask_of(cpu)); + dsb_sev(); + return 0; +} + +static void __cpuinit mcpm_secondary_init(unsigned int cpu) +{ + mcpm_cpu_powered_up(); + gic_secondary_init(0); +} + +#ifdef CONFIG_HOTPLUG_CPU + +static int mcpm_cpu_disable(unsigned int cpu) +{ + /* + * We assume all CPUs may be shut down. + * This would be the hook to use for eventual Secure + * OS migration requests as described in the PSCI spec. + */ + return 0; +} + +static void mcpm_cpu_die(unsigned int cpu) +{ + unsigned int mpidr, pcpu, pcluster; + mpidr = read_cpuid_mpidr(); + pcpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); + pcluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); + mcpm_set_entry_vector(pcpu, pcluster, NULL); + mcpm_cpu_power_down(); +} + +#endif + +struct smp_operations __initdata mcpm_smp_ops = { + .smp_init_cpus = simple_smp_init_cpus, + .smp_boot_secondary = mcpm_boot_secondary, + .smp_secondary_init = mcpm_secondary_init, +#ifdef CONFIG_HOTPLUG_CPU + .cpu_disable = mcpm_cpu_disable, + .cpu_die = mcpm_cpu_die, +#endif +}; -- cgit v1.2.3 From a7eb7c6f9a657a01a8359edae31bbeacd18b072c Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 9 Apr 2013 01:29:17 -0400 Subject: ARM: mcpm: provide an interface to set the SMP ops at run time This is cleaner than exporting the mcpm_smp_ops structure. Signed-off-by: Nicolas Pitre Acked-by: Jon Medhurst --- arch/arm/common/mcpm_platsmp.c | 7 ++++++- arch/arm/include/asm/mcpm.h | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/arm/common/mcpm_platsmp.c b/arch/arm/common/mcpm_platsmp.c index e7e3235b8b6..52b88d81b7b 100644 --- a/arch/arm/common/mcpm_platsmp.c +++ b/arch/arm/common/mcpm_platsmp.c @@ -76,7 +76,7 @@ static void mcpm_cpu_die(unsigned int cpu) #endif -struct smp_operations __initdata mcpm_smp_ops = { +static struct smp_operations __initdata mcpm_smp_ops = { .smp_init_cpus = simple_smp_init_cpus, .smp_boot_secondary = mcpm_boot_secondary, .smp_secondary_init = mcpm_secondary_init, @@ -85,3 +85,8 @@ struct smp_operations __initdata mcpm_smp_ops = { .cpu_die = mcpm_cpu_die, #endif }; + +void __init mcpm_smp_set_ops(void) +{ + smp_set_ops(&mcpm_smp_ops); +} diff --git a/arch/arm/include/asm/mcpm.h b/arch/arm/include/asm/mcpm.h index 3046e90210c..0f7b7620e9a 100644 --- a/arch/arm/include/asm/mcpm.h +++ b/arch/arm/include/asm/mcpm.h @@ -167,6 +167,8 @@ int __mcpm_cluster_state(unsigned int cluster); int __init mcpm_sync_init( void (*power_up_setup)(unsigned int affinity_level)); +void __init mcpm_smp_set_ops(void); + #else /* -- cgit v1.2.3 From 77191d82512a44533bb13d49627c62f1562fc02a Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Wed, 20 Feb 2013 17:34:20 +0000 Subject: ARM: mcpm: Make all mcpm functions notrace The functions in mcpm_entry.c are mostly intended for use during scary cache and coherency disabling sequences, or do other things which confuse trace ... like powering a CPU down and not returning. Similarly for the backend code. For simplicity, this patch just makes whole files notrace. There should be more than enough traceable points on the paths to these functions, but we can be more fine-grained later if there is a need for it. Signed-off-by: Dave Martin --- arch/arm/common/Makefile | 1 + 1 file changed, 1 insertion(+) (limited to 'arch') diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile index 48c0ee57817..bd48ab52544 100644 --- a/arch/arm/common/Makefile +++ b/arch/arm/common/Makefile @@ -12,3 +12,4 @@ obj-$(CONFIG_SHARP_SCOOP) += scoop.o obj-$(CONFIG_PCI_HOST_ITE8152) += it8152.o obj-$(CONFIG_ARM_TIMER_SP804) += timer-sp.o obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o mcpm_platsmp.o vlock.o +CFLAGS_REMOVE_mcpm_entry.o = -pg -- cgit v1.2.3 From fd9b572ce3bf0b9f9aaa000d7fbbcebb17608f53 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 19 Mar 2013 16:07:17 -0400 Subject: ARM: introduce a standalone CONFIG_BIG_LITTLE option The BIG_LITTLE config option is independent from the MCPM one. Semantically, they cover different things. MCPM can be used on non b.L systems for example. Signed-of-by: Nicolas Pitre Signed-off-by: Jon Medhurst --- arch/arm/Kconfig | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'arch') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e5260d4a81f..a8f14c03907 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1614,6 +1614,13 @@ config MCPM for (multi-)cluster based systems, such as big.LITTLE based systems. +config BIG_LITTLE + bool "big.LITTLE support (Experimental)" + depends on CPU_V7 && SMP + select MCPM + help + This option enables support for the big.LITTLE architecture. + choice prompt "Memory split" default VMSPLIT_3G -- cgit v1.2.3 From 25df780c82b9121aa1d38f16f4c470dd3375dfde Mon Sep 17 00:00:00 2001 From: Jon Medhurst Date: Thu, 13 Dec 2012 13:23:13 +0000 Subject: ARM: Enable selection of SMP operations at boot time Add a new 'smp_init' hook to machine_desc so platforms can specify a function to be used to setup smp ops instead of having a statically defined value. The hook must return true when smp_ops are initialized. If false the static mdesc->smp_ops will be used by default. Add the definition of "bool" by including the linux/types.h file to asm/mach/arch.h and make it self-contained. Signed-off-by: Jon Medhurst Signed-off-by: Nicolas Pitre Signed-off-by: Stefano Stabellini Signed-off-by: Nicolas Ferre --- arch/arm/include/asm/mach/arch.h | 5 +++++ arch/arm/kernel/setup.c | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index 308ad7d6f98..75bf07910b8 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -8,6 +8,8 @@ * published by the Free Software Foundation. */ +#include + #ifndef __ASSEMBLY__ struct tag; @@ -16,8 +18,10 @@ struct pt_regs; struct smp_operations; #ifdef CONFIG_SMP #define smp_ops(ops) (&(ops)) +#define smp_init_ops(ops) (&(ops)) #else #define smp_ops(ops) (struct smp_operations *)NULL +#define smp_init_ops(ops) (bool (*)(void))NULL #endif struct machine_desc { @@ -41,6 +45,7 @@ struct machine_desc { unsigned char reserve_lp2 :1; /* never has lp2 */ char restart_mode; /* default restart mode */ struct smp_operations *smp; /* SMP operations */ + bool (*smp_init)(void); void (*fixup)(struct tag *, char **, struct meminfo *); void (*reserve)(void);/* reserve mem blocks */ diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 234e339196c..bce757c97b2 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -787,7 +787,10 @@ void __init setup_arch(char **cmdline_p) arm_dt_init_cpu_maps(); #ifdef CONFIG_SMP if (is_smp()) { - smp_set_ops(mdesc->smp); + if (!mdesc->smp_init || !mdesc->smp_init()) { + if(mdesc->smp) + smp_set_ops(mdesc->smp); + } smp_init_cpus(); } #endif -- cgit v1.2.3 From 01f069610e04c49973e5bbafe799c99e4752822e Mon Sep 17 00:00:00 2001 From: Jon Medhurst Date: Wed, 30 Jan 2013 09:12:55 +0000 Subject: ARM: vexpress: Select multi-cluster SMP operation if required Signed-off-by: Jon Medhurst --- arch/arm/mach-vexpress/core.h | 2 ++ arch/arm/mach-vexpress/platsmp.c | 13 +++++++++++++ arch/arm/mach-vexpress/v2m.c | 1 + 3 files changed, 16 insertions(+) (limited to 'arch') diff --git a/arch/arm/mach-vexpress/core.h b/arch/arm/mach-vexpress/core.h index f134cd4a85f..bde4374ab6d 100644 --- a/arch/arm/mach-vexpress/core.h +++ b/arch/arm/mach-vexpress/core.h @@ -6,6 +6,8 @@ void vexpress_dt_smp_map_io(void); +bool vexpress_smp_init_ops(void); + extern struct smp_operations vexpress_smp_ops; extern void vexpress_cpu_die(unsigned int cpu); diff --git a/arch/arm/mach-vexpress/platsmp.c b/arch/arm/mach-vexpress/platsmp.c index dc1ace55d55..21368ba6ca2 100644 --- a/arch/arm/mach-vexpress/platsmp.c +++ b/arch/arm/mach-vexpress/platsmp.c @@ -12,9 +12,11 @@ #include #include #include +#include #include #include +#include #include #include @@ -203,3 +205,14 @@ struct smp_operations __initdata vexpress_smp_ops = { .cpu_die = vexpress_cpu_die, #endif }; + +bool __init vexpress_smp_init_ops(void) +{ +#ifdef CONFIG_MCPM + if(of_find_compatible_node(NULL, NULL, "arm,cci")) { + mcpm_smp_set_ops(); + return true; + } +#endif + return false; +} diff --git a/arch/arm/mach-vexpress/v2m.c b/arch/arm/mach-vexpress/v2m.c index 915683cb67d..16b42c10e0d 100644 --- a/arch/arm/mach-vexpress/v2m.c +++ b/arch/arm/mach-vexpress/v2m.c @@ -476,6 +476,7 @@ static const char * const v2m_dt_match[] __initconst = { DT_MACHINE_START(VEXPRESS_DT, "ARM-Versatile Express") .dt_compat = v2m_dt_match, .smp = smp_ops(vexpress_smp_ops), + .smp_init = smp_init_ops(vexpress_smp_init_ops), .map_io = v2m_dt_map_io, .init_early = v2m_dt_init_early, .init_irq = irqchip_init, -- cgit v1.2.3 From 2ce16b32ad5d5d69591ea9fe9917c8fe5598a22c Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 19 Mar 2013 22:43:53 -0400 Subject: ARM: mcpm_platsmp: no need to call gic_secondary_init anymore Signed-off-by: Jon Medhurst --- arch/arm/common/mcpm_platsmp.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'arch') diff --git a/arch/arm/common/mcpm_platsmp.c b/arch/arm/common/mcpm_platsmp.c index 52b88d81b7b..3caed0db698 100644 --- a/arch/arm/common/mcpm_platsmp.c +++ b/arch/arm/common/mcpm_platsmp.c @@ -15,8 +15,6 @@ #include #include -#include - #include #include #include @@ -49,7 +47,6 @@ static int __cpuinit mcpm_boot_secondary(unsigned int cpu, struct task_struct *i static void __cpuinit mcpm_secondary_init(unsigned int cpu) { mcpm_cpu_powered_up(); - gic_secondary_init(0); } #ifdef CONFIG_HOTPLUG_CPU -- cgit v1.2.3