Skip to content

Commit

Permalink
perf/x86: Hybrid PMU support for counters
Browse files Browse the repository at this point in the history 
The number of GP and fixed counters are different among hybrid PMUs.
Each hybrid PMU should use its own counter related information.

When handling a certain hybrid PMU, apply the number of counters from
the corresponding hybrid PMU.

When reserving the counters in the initialization of a new event,
reserve all possible counters.

The number of counter recored in the global x86_pmu is for the
architecture counters which are available for all hybrid PMUs. KVM
doesn't support the hybrid PMU yet. Return the number of the
architecture counters for now.

For the functions only available for the old platforms, e.g.,
intel_pmu_drain_pebs_nhm(), nothing is changed.

Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Andi Kleen <ak@linux.intel.com>
Link: https://lkml.kernel.org/r/1618237865-33448-7-git-send-email-kan.liang@linux.intel.com
  • Loading branch information
Kan Liang authored and Peter Zijlstra committed Apr 19, 2021
1 parent fc4b8fc commit d4b294b
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 25 deletions.
55 changes: 38 additions & 17 deletions arch/x86/events/core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,16 +185,29 @@ static DEFINE_MUTEX(pmc_reserve_mutex);

#ifdef CONFIG_X86_LOCAL_APIC

static inline int get_possible_num_counters(void)
{
int i, num_counters = x86_pmu.num_counters;

if (!is_hybrid())
return num_counters;

for (i = 0; i < x86_pmu.num_hybrid_pmus; i++)
num_counters = max_t(int, num_counters, x86_pmu.hybrid_pmu[i].num_counters);

return num_counters;
}

static bool reserve_pmc_hardware(void)
{
int i;
int i, num_counters = get_possible_num_counters();

for (i = 0; i < x86_pmu.num_counters; i++) {
for (i = 0; i < num_counters; i++) {
if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
goto perfctr_fail;
}

for (i = 0; i < x86_pmu.num_counters; i++) {
for (i = 0; i < num_counters; i++) {
if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
goto eventsel_fail;
}
Expand All @@ -205,7 +218,7 @@ static bool reserve_pmc_hardware(void)
for (i--; i >= 0; i--)
release_evntsel_nmi(x86_pmu_config_addr(i));

i = x86_pmu.num_counters;
i = num_counters;

perfctr_fail:
for (i--; i >= 0; i--)
Expand All @@ -216,9 +229,9 @@ static bool reserve_pmc_hardware(void)

static void release_pmc_hardware(void)
{
int i;
int i, num_counters = get_possible_num_counters();

for (i = 0; i < x86_pmu.num_counters; i++) {
for (i = 0; i < num_counters; i++) {
release_perfctr_nmi(x86_pmu_event_addr(i));
release_evntsel_nmi(x86_pmu_config_addr(i));
}
Expand Down Expand Up @@ -946,6 +959,7 @@ EXPORT_SYMBOL_GPL(perf_assign_events);

int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
{
int num_counters = hybrid(cpuc->pmu, num_counters);
struct event_constraint *c;
struct perf_event *e;
int n0, i, wmin, wmax, unsched = 0;
Expand Down Expand Up @@ -1021,7 +1035,7 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)

/* slow path */
if (i != n) {
int gpmax = x86_pmu.num_counters;
int gpmax = num_counters;

/*
* Do not allow scheduling of more than half the available
Expand All @@ -1042,7 +1056,7 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
* the extra Merge events needed by large increment events.
*/
if (x86_pmu.flags & PMU_FL_PAIR) {
gpmax = x86_pmu.num_counters - cpuc->n_pair;
gpmax = num_counters - cpuc->n_pair;
WARN_ON(gpmax <= 0);
}

Expand Down Expand Up @@ -1129,10 +1143,12 @@ static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event,
*/
static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
{
int num_counters = hybrid(cpuc->pmu, num_counters);
int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);
struct perf_event *event;
int n, max_count;

max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
max_count = num_counters + num_counters_fixed;

/* current number of events already accepted */
n = cpuc->n_events;
Expand Down Expand Up @@ -1499,18 +1515,18 @@ void perf_event_print_debug(void)
{
u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
u64 pebs, debugctl;
struct cpu_hw_events *cpuc;
int cpu = smp_processor_id();
struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
int num_counters = hybrid(cpuc->pmu, num_counters);
int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);
unsigned long flags;
int cpu, idx;
int idx;

if (!x86_pmu.num_counters)
if (!num_counters)
return;

local_irq_save(flags);

cpu = smp_processor_id();
cpuc = &per_cpu(cpu_hw_events, cpu);

if (x86_pmu.version >= 2) {
rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
Expand All @@ -1533,7 +1549,7 @@ void perf_event_print_debug(void)
}
pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);

for (idx = 0; idx < x86_pmu.num_counters; idx++) {
for (idx = 0; idx < num_counters; idx++) {
rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
rdmsrl(x86_pmu_event_addr(idx), pmc_count);

Expand All @@ -1546,7 +1562,7 @@ void perf_event_print_debug(void)
pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
cpu, idx, prev_left);
}
for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
for (idx = 0; idx < num_counters_fixed; idx++) {
if (fixed_counter_disabled(idx, cpuc->pmu))
continue;
rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
Expand Down Expand Up @@ -2781,6 +2797,11 @@ unsigned long perf_misc_flags(struct pt_regs *regs)
void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
{
cap->version = x86_pmu.version;
/*
* KVM doesn't support the hybrid PMU yet.
* Return the common value in global x86_pmu,
* which available for all cores.
*/
cap->num_counters_gp = x86_pmu.num_counters;
cap->num_counters_fixed = x86_pmu.num_counters_fixed;
cap->bit_width_gp = x86_pmu.cntval_bits;
Expand Down
8 changes: 5 additions & 3 deletions arch/x86/events/intel/core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2711,21 +2711,23 @@ static void intel_pmu_reset(void)
{
struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);
int num_counters = hybrid(cpuc->pmu, num_counters);
unsigned long flags;
int idx;

if (!x86_pmu.num_counters)
if (!num_counters)
return;

local_irq_save(flags);

pr_info("clearing PMU state on CPU#%d\n", smp_processor_id());

for (idx = 0; idx < x86_pmu.num_counters; idx++) {
for (idx = 0; idx < num_counters; idx++) {
wrmsrl_safe(x86_pmu_config_addr(idx), 0ull);
wrmsrl_safe(x86_pmu_event_addr(idx), 0ull);
}
for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
for (idx = 0; idx < num_counters_fixed; idx++) {
if (fixed_counter_disabled(idx, cpuc->pmu))
continue;
wrmsrl_safe(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
Expand Down
14 changes: 9 additions & 5 deletions arch/x86/events/intel/ds.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1007,16 +1007,18 @@ void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in)
static inline void pebs_update_threshold(struct cpu_hw_events *cpuc)
{
struct debug_store *ds = cpuc->ds;
int max_pebs_events = hybrid(cpuc->pmu, max_pebs_events);
int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);
u64 threshold;
int reserved;

if (cpuc->n_pebs_via_pt)
return;

if (x86_pmu.flags & PMU_FL_PEBS_ALL)
reserved = x86_pmu.max_pebs_events + x86_pmu.num_counters_fixed;
reserved = max_pebs_events + num_counters_fixed;
else
reserved = x86_pmu.max_pebs_events;
reserved = max_pebs_events;

if (cpuc->n_pebs == cpuc->n_large_pebs) {
threshold = ds->pebs_absolute_maximum -
Expand Down Expand Up @@ -2072,6 +2074,8 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
{
short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
int max_pebs_events = hybrid(cpuc->pmu, max_pebs_events);
int num_counters_fixed = hybrid(cpuc->pmu, num_counters_fixed);
struct debug_store *ds = cpuc->ds;
struct perf_event *event;
void *base, *at, *top;
Expand All @@ -2086,9 +2090,9 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d

ds->pebs_index = ds->pebs_buffer_base;

mask = ((1ULL << x86_pmu.max_pebs_events) - 1) |
(((1ULL << x86_pmu.num_counters_fixed) - 1) << INTEL_PMC_IDX_FIXED);
size = INTEL_PMC_IDX_FIXED + x86_pmu.num_counters_fixed;
mask = ((1ULL << max_pebs_events) - 1) |
(((1ULL << num_counters_fixed) - 1) << INTEL_PMC_IDX_FIXED);
size = INTEL_PMC_IDX_FIXED + num_counters_fixed;

if (unlikely(base >= top)) {
intel_pmu_pebs_event_update_no_drain(cpuc, size);
Expand Down
4 changes: 4 additions & 0 deletions arch/x86/events/perf_event.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -635,6 +635,9 @@ struct x86_hybrid_pmu {
struct pmu pmu;
union perf_capabilities intel_cap;
u64 intel_ctrl;
int max_pebs_events;
int num_counters;
int num_counters_fixed;
};

static __always_inline struct x86_hybrid_pmu *hybrid_pmu(struct pmu *pmu)
Expand Down Expand Up @@ -850,6 +853,7 @@ struct x86_pmu {
* are available for all PMUs. The hybrid_pmu only includes the
* unique capabilities.
*/
int num_hybrid_pmus;
struct x86_hybrid_pmu *hybrid_pmu;
};

Expand Down

0 comments on commit d4b294b

Please sign in to comment.