Support CCI PMU in perf

CCI400 has a set of counters that can be used to profile different
transations at CCI master and slave interfaces. These counters can
observe different kinds of transations passing through the CCI and
provide a system-level view of activity.

This patch adds support for CCI PMU by extending the existing CCI
driver.

Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Signed-off-by: Jon Medhurst <tixy@linaro.org>

1 files changed, 364 insertions, 1 deletions

diff --git a/drivers/bus/arm-cci.c b/drivers/bus/arm-cci.c
index a621d3d8b90..ed284ab0041 100644
--- a/drivers/bus/arm-cci.c
+++ b/drivers/bus/arm-cci.c
@@ -22,6 +22,8 @@
 #include <linux/slab.h>
 
 #include <asm/cacheflush.h>
+#include <asm/irq_regs.h>
+#include <asm/pmu.h>
 #include <asm/smp_plat.h>
 
 #define DRIVER_NAME		"CCI"
@@ -61,6 +63,362 @@ static unsigned long cci_ctrl_phys;
 
 static void __iomem *cci_pmu_base;
 
+#define CCI400_PMCR		0x0100
+
+#define CCI400_PMU_CYCLE_CNTR_BASE    0x0000
+#define CCI400_PMU_CNTR_BASE(idx)     (CCI400_PMU_CYCLE_CNTR_BASE + (idx) * 0x1000)
+
+#define CCI400_PMCR_CEN          0x00000001
+#define CCI400_PMCR_RST          0x00000002
+#define CCI400_PMCR_CCR          0x00000004
+#define CCI400_PMCR_CCD          0x00000008
+#define CCI400_PMCR_EX           0x00000010
+#define CCI400_PMCR_DP           0x00000020
+#define CCI400_PMCR_NCNT_MASK    0x0000F800
+#define CCI400_PMCR_NCNT_SHIFT   11
+
+#define CCI400_PMU_EVT_SEL       0x000
+#define CCI400_PMU_CNTR          0x004
+#define CCI400_PMU_CNTR_CTRL     0x008
+#define CCI400_PMU_OVERFLOW      0x00C
+
+#define CCI400_PMU_OVERFLOW_FLAG 1
+
+enum cci400_perf_events {
+	CCI400_PMU_CYCLES = 0xFF
+};
+
+#define CCI400_PMU_EVENT_MASK   0xff
+#define CCI400_PMU_EVENT_SOURCE(event) ((event >> 5) & 0x7)
+#define CCI400_PMU_EVENT_CODE(event) (event & 0x1f)
+
+#define CCI400_PMU_EVENT_SOURCE_S0 0
+#define CCI400_PMU_EVENT_SOURCE_S4 4
+#define CCI400_PMU_EVENT_SOURCE_M0 5
+#define CCI400_PMU_EVENT_SOURCE_M2 7
+
+#define CCI400_PMU_EVENT_SLAVE_MIN 0x0
+#define CCI400_PMU_EVENT_SLAVE_MAX 0x13
+
+#define CCI400_PMU_EVENT_MASTER_MIN 0x14
+#define CCI400_PMU_EVENT_MASTER_MAX 0x1A
+
+#define CCI400_PMU_MAX_HW_EVENTS 5   /* CCI PMU has 4 counters + 1 cycle counter */
+
+#define CCI400_PMU_CYCLE_COUNTER_IDX 0
+#define CCI400_PMU_COUNTER0_IDX      1
+#define CCI400_PMU_COUNTER_LAST(cci_pmu) (CCI400_PMU_CYCLE_COUNTER_IDX + cci_pmu->num_events - 1)
+
+
+static struct perf_event *events[CCI400_PMU_MAX_HW_EVENTS];
+static unsigned long used_mask[BITS_TO_LONGS(CCI400_PMU_MAX_HW_EVENTS)];
+static struct pmu_hw_events cci_hw_events = {
+	.events    = events,
+	.used_mask = used_mask,
+};
+
+static int cci_pmu_validate_hw_event(u8 hw_event)
+{
+	u8 ev_source = CCI400_PMU_EVENT_SOURCE(hw_event);
+	u8 ev_code = CCI400_PMU_EVENT_CODE(hw_event);
+
+	if (ev_source <= CCI400_PMU_EVENT_SOURCE_S4 &&
+	    ev_code <= CCI400_PMU_EVENT_SLAVE_MAX)
+			return hw_event;
+	else if (CCI400_PMU_EVENT_SOURCE_M0 <= ev_source &&
+		   ev_source <= CCI400_PMU_EVENT_SOURCE_M2 &&
+		   CCI400_PMU_EVENT_MASTER_MIN <= ev_code &&
+		    ev_code <= CCI400_PMU_EVENT_MASTER_MAX)
+			return hw_event;
+
+	return -EINVAL;
+}
+
+static inline int cci_pmu_counter_is_valid(struct arm_pmu *cci_pmu, int idx)
+{
+	return CCI400_PMU_CYCLE_COUNTER_IDX <= idx &&
+		idx <= CCI400_PMU_COUNTER_LAST(cci_pmu);
+}
+
+static inline u32 cci_pmu_read_register(int idx, unsigned int offset)
+{
+	return readl_relaxed(cci_pmu_base + CCI400_PMU_CNTR_BASE(idx) + offset);
+}
+
+static inline void cci_pmu_write_register(u32 value, int idx, unsigned int offset)
+{
+	return writel_relaxed(value, cci_pmu_base + CCI400_PMU_CNTR_BASE(idx) + offset);
+}
+
+static inline void cci_pmu_disable_counter(int idx)
+{
+	cci_pmu_write_register(0, idx, CCI400_PMU_CNTR_CTRL);
+}
+
+static inline void cci_pmu_enable_counter(int idx)
+{
+	cci_pmu_write_register(1, idx, CCI400_PMU_CNTR_CTRL);
+}
+
+static inline void cci_pmu_select_event(int idx, unsigned long event)
+{
+	event &= CCI400_PMU_EVENT_MASK;
+	cci_pmu_write_register(event, idx, CCI400_PMU_EVT_SEL);
+}
+
+static u32 cci_pmu_get_max_counters(void)
+{
+	u32 n_cnts = (readl_relaxed(cci_ctrl_base + CCI400_PMCR) &
+		      CCI400_PMCR_NCNT_MASK) >> CCI400_PMCR_NCNT_SHIFT;
+
+	/* add 1 for cycle counter */
+	return n_cnts + 1;
+}
+
+static struct pmu_hw_events *cci_pmu_get_hw_events(void)
+{
+	return &cci_hw_events;
+}
+
+static int cci_pmu_get_event_idx(struct pmu_hw_events *hw, struct perf_event *event)
+{
+	struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
+	struct hw_perf_event *hw_event = &event->hw;
+	unsigned long cci_event = hw_event->config_base & CCI400_PMU_EVENT_MASK;
+	int idx;
+
+	if (cci_event == CCI400_PMU_CYCLES) {
+		if (test_and_set_bit(CCI400_PMU_CYCLE_COUNTER_IDX, hw->used_mask))
+			return -EAGAIN;
+
+                return CCI400_PMU_CYCLE_COUNTER_IDX;
+        }
+
+	for (idx = CCI400_PMU_COUNTER0_IDX; idx <= CCI400_PMU_COUNTER_LAST(cci_pmu); ++idx) {
+		if (!test_and_set_bit(idx, hw->used_mask))
+			return idx;
+	}
+
+	/* No counters available */
+	return -EAGAIN;
+}
+
+static int cci_pmu_map_event(struct perf_event *event)
+{
+	int mapping;
+	u8 config = event->attr.config & CCI400_PMU_EVENT_MASK;
+
+	if (event->attr.type < PERF_TYPE_MAX)
+		return -ENOENT;
+
+	/* 0xff is used to represent CCI Cycles */
+	if (config == 0xff)
+		mapping = config;
+	else
+		mapping = cci_pmu_validate_hw_event(config);
+
+	return mapping;
+}
+
+static int cci_pmu_request_irq(struct arm_pmu *cci_pmu, irq_handler_t handler)
+{
+	int irq, err, i = 0;
+	struct platform_device *pmu_device = cci_pmu->plat_device;
+
+	if (unlikely(!pmu_device))
+		return -ENODEV;
+
+	/* CCI exports 6 interrupts - 1 nERRORIRQ + 5 nEVNTCNTOVERFLOW (PMU)
+	   nERRORIRQ will be handled by secure firmware on TC2. So we
+	   assume that all CCI interrupts listed in the linux device
+	   tree are PMU interrupts.
+
+	   The following code should then be able to handle different routing
+	   of the CCI PMU interrupts.
+	*/
+	while ((irq = platform_get_irq(pmu_device, i)) > 0) {
+		err = request_irq(irq, handler, 0, "arm-cci-pmu", cci_pmu);
+		if (err) {
+			dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n",
+				irq);
+			return err;
+		}
+		i++;
+	}
+
+	return 0;
+}
+
+static irqreturn_t cci_pmu_handle_irq(int irq_num, void *dev)
+{
+	struct arm_pmu *cci_pmu = (struct arm_pmu *)dev;
+	struct pmu_hw_events *events = cci_pmu->get_hw_events();
+	struct perf_sample_data data;
+	struct pt_regs *regs;
+	int idx;
+
+	regs = get_irq_regs();
+
+	/* Iterate over counters and update the corresponding perf events.
+	   This should work regardless of whether we have per-counter overflow
+	   interrupt or a combined overflow interrupt. */
+	for (idx = CCI400_PMU_CYCLE_COUNTER_IDX; idx <= CCI400_PMU_COUNTER_LAST(cci_pmu); idx++) {
+		struct perf_event *event = events->events[idx];
+		struct hw_perf_event *hw_counter;
+
+		if (!event)
+			continue;
+
+		hw_counter = &event->hw;
+
+		/* Did this counter overflow? */
+		if (!(cci_pmu_read_register(idx, CCI400_PMU_OVERFLOW) & CCI400_PMU_OVERFLOW_FLAG))
+			continue;
+		cci_pmu_write_register(CCI400_PMU_OVERFLOW_FLAG, idx, CCI400_PMU_OVERFLOW);
+
+		armpmu_event_update(event);
+		perf_sample_data_init(&data, 0, hw_counter->last_period);
+		if (!armpmu_event_set_period(event))
+			continue;
+
+		if (perf_event_overflow(event, &data, regs))
+			cci_pmu->disable(event);
+	}
+
+	irq_work_run();
+	return IRQ_HANDLED;
+}
+
+static void cci_pmu_free_irq(struct arm_pmu *cci_pmu)
+{
+	int irq, i = 0;
+	struct platform_device *pmu_device = cci_pmu->plat_device;
+
+	while ((irq = platform_get_irq(pmu_device, i)) > 0) {
+		free_irq(irq, cci_pmu);
+		i++;
+	}
+}
+
+static void cci_pmu_enable_event(struct perf_event *event)
+{
+	unsigned long flags;
+	struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
+	struct pmu_hw_events *events = cci_pmu->get_hw_events();
+	struct hw_perf_event *hw_counter = &event->hw;
+	int idx = hw_counter->idx;
+
+	if (unlikely(!cci_pmu_counter_is_valid(cci_pmu, idx))) {
+		dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
+		return;
+	}
+
+	raw_spin_lock_irqsave(&events->pmu_lock, flags);
+
+	/* Configure the event to count, unless you are counting cycles */
+	if (idx != CCI400_PMU_CYCLE_COUNTER_IDX)
+		cci_pmu_select_event(idx, hw_counter->config_base);
+
+	cci_pmu_enable_counter(idx);
+
+	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
+}
+
+static void cci_pmu_disable_event(struct perf_event *event)
+{
+	unsigned long flags;
+	struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
+	struct pmu_hw_events *events = cci_pmu->get_hw_events();
+	struct hw_perf_event *hw_counter = &event->hw;
+	int idx = hw_counter->idx;
+
+	if (unlikely(!cci_pmu_counter_is_valid(cci_pmu, idx))) {
+		dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
+		return;
+	}
+
+	raw_spin_lock_irqsave(&events->pmu_lock, flags);
+
+	cci_pmu_disable_counter(idx);
+
+	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
+}
+
+static void cci_pmu_start(struct arm_pmu *cci_pmu)
+{
+	u32 val;
+	unsigned long flags;
+	struct pmu_hw_events *events = cci_pmu->get_hw_events();
+
+	raw_spin_lock_irqsave(&events->pmu_lock, flags);
+
+	/* Enable all the PMU counters. */
+	val = readl(cci_ctrl_base + CCI400_PMCR) | CCI400_PMCR_CEN;
+	writel(val, cci_ctrl_base + CCI400_PMCR);
+
+	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
+}
+
+static void cci_pmu_stop(struct arm_pmu *cci_pmu)
+{
+	u32 val;
+	unsigned long flags;
+	struct pmu_hw_events *events = cci_pmu->get_hw_events();
+
+	raw_spin_lock_irqsave(&events->pmu_lock, flags);
+
+	/* Disable all the PMU counters. */
+	val = readl(cci_ctrl_base + CCI400_PMCR) & ~CCI400_PMCR_CEN;
+	writel(val, cci_ctrl_base + CCI400_PMCR);
+
+	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
+}
+
+static u32 cci_pmu_read_counter(struct perf_event *event)
+{
+	struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
+	struct hw_perf_event *hw_counter = &event->hw;
+	int idx = hw_counter->idx;
+	u32 value;
+
+	if (unlikely(!cci_pmu_counter_is_valid(cci_pmu, idx))) {
+		dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
+		return 0;
+	}
+	value = cci_pmu_read_register(idx, CCI400_PMU_CNTR);
+
+	return value;
+}
+
+static void cci_pmu_write_counter(struct perf_event *event, u32 value)
+{
+	struct arm_pmu *cci_pmu = to_arm_pmu(event->pmu);
+	struct hw_perf_event *hw_counter = &event->hw;
+	int idx = hw_counter->idx;
+
+	if (unlikely(!cci_pmu_counter_is_valid(cci_pmu, idx)))
+		dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
+	else
+		cci_pmu_write_register(value, idx, CCI400_PMU_CNTR);
+}
+
+static struct arm_pmu cci_pmu = {
+	.name             = DRIVER_NAME,
+	.max_period       = (1LLU << 32) - 1,
+	.get_hw_events    = cci_pmu_get_hw_events,
+	.get_event_idx    = cci_pmu_get_event_idx,
+	.map_event        = cci_pmu_map_event,
+	.request_irq      = cci_pmu_request_irq,
+	.handle_irq       = cci_pmu_handle_irq,
+	.free_irq         = cci_pmu_free_irq,
+	.enable           = cci_pmu_enable_event,
+	.disable          = cci_pmu_disable_event,
+	.start            = cci_pmu_start,
+	.stop             = cci_pmu_stop,
+	.read_counter     = cci_pmu_read_counter,
+	.write_counter    = cci_pmu_write_counter,
+};
+
 static int cci_pmu_probe(struct platform_device *pdev)
 {
 	struct resource *res;
@@ -70,7 +428,12 @@ static int cci_pmu_probe(struct platform_device *pdev)
 	if (IS_ERR(cci_pmu_base))
 		return PTR_ERR(cci_pmu_base);
 
-	return 0;
+	cci_pmu.plat_device = pdev;
+	cci_pmu.num_events = cci_pmu_get_max_counters();
+	raw_spin_lock_init(&cci_hw_events.pmu_lock);
+	cpumask_setall(&cci_pmu.valid_cpus);
+
+	return armpmu_register(&cci_pmu, -1);
 }
 
 static const struct of_device_id arm_cci_pmu_matches[] = {