static void p4_check_ctrs(unsigned int const cpu, struct op_msrs const * const msrs, struct pt_regs * const regs) { unsigned long ctr, low, high, stag, real; int i; stag = get_stagger(); for (i = 0; i < num_counters; ++i) { if (!sysctl.ctr[i].enabled) continue; real = VIRT_CTR(stag, i); CCCR_READ(low, high, real); CTR_READ(ctr, high, real); if (CCCR_OVF_P(low) || CTR_OVERFLOW_P(ctr)) { op_do_profile(cpu, instruction_pointer(regs), IRQ_ENABLED(regs), i); CTR_WRITE(oprof_data[cpu].ctr_count[i], real); CCCR_CLEAR_OVF(low); CCCR_WRITE(low, high, real); CTR_WRITE(oprof_data[cpu].ctr_count[i], real); } } apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED); }
static void pmc_setup_one_p4_counter(unsigned int ctr) { int i; int const maxbind = 2; unsigned int cccr = 0; unsigned int escr = 0; unsigned int high = 0; unsigned int counter_bit; struct p4_event_binding * ev = 0; unsigned int stag; stag = get_stagger(); /* convert from counter *number* to counter *bit* */ counter_bit = 1 << VIRT_CTR(stag, ctr); /* find our event binding structure. */ if (counter_config[ctr].event <= 0 || counter_config[ctr].event > NUM_EVENTS) { printk(KERN_ERR "oprofile: P4 event code 0x%lx out of range\n", counter_config[ctr].event); return; } ev = &(p4_events[counter_config[ctr].event - 1]); for (i = 0; i < maxbind; i++) { if (ev->bindings[i].virt_counter & counter_bit) { /* modify ESCR */ ESCR_READ(escr, high, ev, i); ESCR_CLEAR(escr); if (stag == 0) { ESCR_SET_USR_0(escr, counter_config[ctr].user); ESCR_SET_OS_0(escr, counter_config[ctr].kernel); } else { ESCR_SET_USR_1(escr, counter_config[ctr].user); ESCR_SET_OS_1(escr, counter_config[ctr].kernel); } ESCR_SET_EVENT_SELECT(escr, ev->event_select); ESCR_SET_EVENT_MASK(escr, counter_config[ctr].unit_mask); ESCR_WRITE(escr, high, ev, i); /* modify CCCR */ CCCR_READ(cccr, high, VIRT_CTR(stag, ctr)); CCCR_CLEAR(cccr); CCCR_SET_REQUIRED_BITS(cccr); CCCR_SET_ESCR_SELECT(cccr, ev->escr_select); if (stag == 0) { CCCR_SET_PMI_OVF_0(cccr); } else { CCCR_SET_PMI_OVF_1(cccr); } CCCR_WRITE(cccr, high, VIRT_CTR(stag, ctr)); return; } } }
static void p4_fill_in_addresses(struct op_msrs * const msrs) { unsigned int i; unsigned int addr, stag; setup_num_counters(); stag = get_stagger(); /* the counter registers we pay attention to */ for (i = 0; i < num_counters; ++i) { msrs->counters.addrs[i] = p4_counters[VIRT_CTR(stag, i)].counter_address; } /* FIXME: bad feeling, we don't save the 10 counters we don't use. */ /* 18 CCCR registers */ for (i = 0, addr = MSR_P4_BPU_CCCR0 + stag; addr <= MSR_P4_IQ_CCCR5; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } /* 43 ESCR registers in three discontiguous group */ for (addr = MSR_P4_BSU_ESCR0 + stag; addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } for (addr = MSR_P4_MS_ESCR0 + stag; addr <= MSR_P4_TC_ESCR1; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } for (addr = MSR_P4_IX_ESCR0 + stag; addr <= MSR_P4_CRU_ESCR3; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } /* there are 2 remaining non-contiguously located ESCRs */ if (num_counters == NUM_COUNTERS_NON_HT) { /* standard non-HT CPUs handle both remaining ESCRs*/ msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR5; msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR4; } else if (stag == 0) { /* HT CPUs give the first remainder to the even thread, as the 32nd control register */ msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR4; } else { /* and two copies of the second to the odd thread, for the 31st and 32nd control registers */ msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR5; msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR5; } }
static int p4_check_ctrs(unsigned int const cpu, struct op_msrs const * const msrs, struct pt_regs * const regs) { unsigned long ctr, low, high, stag, real; int i; unsigned long eip = instruction_pointer(regs); int is_kernel = !user_mode(regs); stag = get_stagger(); for (i = 0; i < num_counters; ++i) { if (!counter_config[i].event) continue; /* * there is some eccentricity in the hardware which * requires that we perform 2 extra corrections: * * - check both the CCCR:OVF flag for overflow and the * counter high bit for un-flagged overflows. * * - write the counter back twice to ensure it gets * updated properly. * * the former seems to be related to extra NMIs happening * during the current NMI; the latter is reported as errata * N15 in intel doc 249199-029, pentium 4 specification * update, though their suggested work-around does not * appear to solve the problem. */ real = VIRT_CTR(stag, i); CCCR_READ(low, high, real); CTR_READ(ctr, high, real); if (CCCR_OVF_P(low) || CTR_OVERFLOW_P(ctr)) { oprofile_add_sample(eip, is_kernel, i, cpu); CTR_WRITE(reset_value[i], real); CCCR_CLEAR_OVF(low); CCCR_WRITE(low, high, real); CTR_WRITE(reset_value[i], real); /* P4 quirk: you have to re-unmask the apic vector */ apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED); } } /* P4 quirk: you have to re-unmask the apic vector */ apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED); /* See op_model_ppro.c */ return 1; }
static void p4_stop(struct op_msrs const * const msrs) { unsigned int low, high, stag; int i; stag = get_stagger(); for (i = 0; i < num_counters; ++i) { CCCR_READ(low, high, VIRT_CTR(stag, i)); CCCR_SET_DISABLE(low); CCCR_WRITE(low, high, VIRT_CTR(stag, i)); } }
static void p4_start(struct op_msrs const * const msrs) { unsigned int low, high, stag; int i; stag = get_stagger(); for (i = 0; i < num_counters; ++i) { if (!sysctl.ctr[i].enabled) continue; CCCR_READ(low, high, VIRT_CTR(stag, i)); CCCR_SET_ENABLE(low); CCCR_WRITE(low, high, VIRT_CTR(stag, i)); } }
static void p4_stop(struct op_msrs const * const msrs) { unsigned int low, high, stag; int i; stag = get_stagger(); for (i = 0; i < num_counters; ++i) { if (!reset_value[i]) continue; rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high); CCCR_SET_DISABLE(low); wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high); } }
static void p4_setup_ctrs(struct op_x86_model_spec const *model, struct op_msrs const * const msrs) { unsigned int i; unsigned int low, high; unsigned int stag; stag = get_stagger(); rdmsr(MSR_IA32_MISC_ENABLE, low, high); if (!MISC_PMC_ENABLED_P(low)) { printk(KERN_ERR "oprofile: P4 PMC not available\n"); return; } /* clear the cccrs we will use */ for (i = 0; i < num_counters; i++) { if (unlikely(!msrs->controls[i].addr)) continue; rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high); CCCR_CLEAR(low); CCCR_SET_REQUIRED_BITS(low); wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high); } /* clear all escrs (including those outside our concern) */ for (i = num_counters; i < num_controls; i++) { if (unlikely(!msrs->controls[i].addr)) continue; wrmsr(msrs->controls[i].addr, 0, 0); } /* setup all counters */ for (i = 0; i < num_counters; ++i) { if (counter_config[i].enabled && msrs->controls[i].addr) { reset_value[i] = counter_config[i].count; pmc_setup_one_p4_counter(i); wrmsrl(p4_counters[VIRT_CTR(stag, i)].counter_address, -(u64)counter_config[i].count); } else { reset_value[i] = 0; } } }
static void p4_setup_ctrs(struct op_msrs const * const msrs) { unsigned int i; unsigned int low, high; unsigned int addr; unsigned int stag; stag = get_stagger(); rdmsr(MSR_IA32_MISC_ENABLE, low, high); if (!MISC_PMC_ENABLED_P(low)) { printk(KERN_ERR "oprofile: P4 PMC not available\n"); return; } for (i = 0 ; i < num_counters ; i++) { rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high); CCCR_CLEAR(low); CCCR_SET_REQUIRED_BITS(low); wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high); } for (i = stag ; i < NUM_UNUSED_CCCRS ; i += addr_increment()) { rdmsr(p4_unused_cccr[i], low, high); CCCR_CLEAR(low); CCCR_SET_REQUIRED_BITS(low); wrmsr(p4_unused_cccr[i], low, high); } for (addr = MSR_P4_BSU_ESCR0 + stag; addr < MSR_P4_IQ_ESCR0; addr += addr_increment()) { wrmsr(addr, 0, 0); } if (boot_cpu_data.x86_model < 0x3) { wrmsr(MSR_P4_IQ_ESCR0, 0, 0); wrmsr(MSR_P4_IQ_ESCR1, 0, 0); } for (addr = MSR_P4_RAT_ESCR0 + stag; addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) { wrmsr(addr, 0, 0); } for (addr = MSR_P4_MS_ESCR0 + stag; addr <= MSR_P4_TC_ESCR1; addr += addr_increment()) { wrmsr(addr, 0, 0); } for (addr = MSR_P4_IX_ESCR0 + stag; addr <= MSR_P4_CRU_ESCR3; addr += addr_increment()) { wrmsr(addr, 0, 0); } if (num_counters == NUM_COUNTERS_NON_HT) { wrmsr(MSR_P4_CRU_ESCR4, 0, 0); wrmsr(MSR_P4_CRU_ESCR5, 0, 0); } else if (stag == 0) { wrmsr(MSR_P4_CRU_ESCR4, 0, 0); } else { wrmsr(MSR_P4_CRU_ESCR5, 0, 0); } for (i = 0 ; i < num_counters ; ++i) { if (sysctl.ctr[i].event) { pmc_setup_one_p4_counter(i); CTR_WRITE(sysctl.ctr[i].count, VIRT_CTR(stag, i)); } } }
static void pmc_setup_one_p4_counter(unsigned int ctr) { int i; int const maxbind = 2; unsigned int cccr = 0; unsigned int escr = 0; unsigned int high = 0; unsigned int counter_bit; struct p4_event_binding * ev = NULL; unsigned int stag; stag = get_stagger(); counter_bit = 1 << VIRT_CTR(stag, ctr); if (sysctl.ctr[ctr].event <= 0 || sysctl.ctr[ctr].event > NUM_EVENTS) { printk(KERN_ERR "oprofile: P4 event code 0x%x out of range\n", sysctl.ctr[ctr].event); return; } ev = &(p4_events[sysctl.ctr[ctr].event - 1]); for (i = 0; i < maxbind; i++) { if (ev->bindings[i].virt_counter & counter_bit) { ESCR_READ(escr, high, ev, i); ESCR_CLEAR(escr); if (stag == 0) { ESCR_SET_USR_0(escr, sysctl.ctr[ctr].user); ESCR_SET_OS_0(escr, sysctl.ctr[ctr].kernel); } else { ESCR_SET_USR_1(escr, sysctl.ctr[ctr].user); ESCR_SET_OS_1(escr, sysctl.ctr[ctr].kernel); } ESCR_SET_EVENT_SELECT(escr, ev->event_select); ESCR_SET_EVENT_MASK(escr, sysctl.ctr[ctr].unit_mask); ESCR_WRITE(escr, high, ev, i); CCCR_READ(cccr, high, VIRT_CTR(stag, ctr)); CCCR_CLEAR(cccr); CCCR_SET_REQUIRED_BITS(cccr); CCCR_SET_ESCR_SELECT(cccr, ev->escr_select); if (stag == 0) { CCCR_SET_PMI_OVF_0(cccr); } else { CCCR_SET_PMI_OVF_1(cccr); } CCCR_WRITE(cccr, high, VIRT_CTR(stag, ctr)); return; } } printk(KERN_ERR "oprofile: P4 event code 0x%x no binding, ctr %d\n", sysctl.ctr[ctr].event, ctr); }
static void p4_fill_in_addresses(struct op_msrs * const msrs) { unsigned int i; unsigned int addr, stag; setup_num_counters(); stag = get_stagger(); for (i = 0; i < num_counters; ++i) { msrs->counters.addrs[i] = p4_counters[VIRT_CTR(stag, i)].counter_address; } for (i = 0, addr = MSR_P4_BPU_CCCR0 + stag; addr <= MSR_P4_IQ_CCCR5; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } for (addr = MSR_P4_BSU_ESCR0 + stag; addr < MSR_P4_IQ_ESCR0; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } if (boot_cpu_data.x86_model >= 0x3) { for (addr = MSR_P4_BSU_ESCR0 + stag; addr <= MSR_P4_BSU_ESCR1; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } } else { for (addr = MSR_P4_IQ_ESCR0 + stag; addr <= MSR_P4_IQ_ESCR1; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } } for (addr = MSR_P4_RAT_ESCR0 + stag; addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } for (addr = MSR_P4_MS_ESCR0 + stag; addr <= MSR_P4_TC_ESCR1; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } for (addr = MSR_P4_IX_ESCR0 + stag; addr <= MSR_P4_CRU_ESCR3; ++i, addr += addr_increment()) { msrs->controls.addrs[i] = addr; } if (num_counters == NUM_COUNTERS_NON_HT) { msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR5; msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR4; } else if (stag == 0) { msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR4; } else { msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR5; msrs->controls.addrs[i++] = MSR_P4_CRU_ESCR5; } }
static void p4_setup_ctrs(struct op_msrs const * const msrs) { unsigned int i; unsigned int low, high; unsigned int addr; unsigned int stag; stag = get_stagger(); rdmsr(MSR_IA32_MISC_ENABLE, low, high); if (! MISC_PMC_ENABLED_P(low)) { printk(KERN_ERR "oprofile: P4 PMC not available\n"); return; } /* clear the cccrs we will use */ for (i = 0 ; i < num_counters ; i++) { rdmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high); CCCR_CLEAR(low); CCCR_SET_REQUIRED_BITS(low); wrmsr(p4_counters[VIRT_CTR(stag, i)].cccr_address, low, high); } /* clear cccrs outside our concern */ for (i = stag ; i < NUM_UNUSED_CCCRS ; i += addr_increment()) { rdmsr(p4_unused_cccr[i], low, high); CCCR_CLEAR(low); CCCR_SET_REQUIRED_BITS(low); wrmsr(p4_unused_cccr[i], low, high); } /* clear all escrs (including those outside our concern) */ for (addr = MSR_P4_BSU_ESCR0 + stag; addr < MSR_P4_IQ_ESCR0; addr += addr_increment()) { wrmsr(addr, 0, 0); } /* On older models clear also MSR_P4_IQ_ESCR0/1 */ if (boot_cpu_data.x86_model < 0x3) { wrmsr(MSR_P4_IQ_ESCR0, 0, 0); wrmsr(MSR_P4_IQ_ESCR1, 0, 0); } for (addr = MSR_P4_RAT_ESCR0 + stag; addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) { wrmsr(addr, 0, 0); } for (addr = MSR_P4_MS_ESCR0 + stag; addr <= MSR_P4_TC_ESCR1; addr += addr_increment()){ wrmsr(addr, 0, 0); } for (addr = MSR_P4_IX_ESCR0 + stag; addr <= MSR_P4_CRU_ESCR3; addr += addr_increment()){ wrmsr(addr, 0, 0); } if (num_counters == NUM_COUNTERS_NON_HT) { wrmsr(MSR_P4_CRU_ESCR4, 0, 0); wrmsr(MSR_P4_CRU_ESCR5, 0, 0); } else if (stag == 0) { wrmsr(MSR_P4_CRU_ESCR4, 0, 0); } else { wrmsr(MSR_P4_CRU_ESCR5, 0, 0); } /* setup all counters */ for (i = 0 ; i < num_counters ; ++i) { if (counter_config[i].enabled) { reset_value[i] = counter_config[i].count; pmc_setup_one_p4_counter(i); CTR_WRITE(counter_config[i].count, VIRT_CTR(stag, i)); } else { reset_value[i] = 0; } } }
static void p4_fill_in_addresses(struct op_msrs * const msrs) { unsigned int i; unsigned int addr, stag; setup_num_counters(); stag = get_stagger(); /* the counter registers we pay attention to */ for (i = 0; i < num_counters; ++i) { msrs->counters[i].addr = p4_counters[VIRT_CTR(stag, i)].counter_address; } /* FIXME: bad feeling, we don't save the 10 counters we don't use. */ /* 18 CCCR registers */ for (i = 0, addr = MSR_P4_BPU_CCCR0 + stag; addr <= MSR_P4_IQ_CCCR5; ++i, addr += addr_increment()) { msrs->controls[i].addr = addr; } /* 43 ESCR registers in three or four discontiguous group */ for (addr = MSR_P4_BSU_ESCR0 + stag; addr < MSR_P4_IQ_ESCR0; ++i, addr += addr_increment()) { msrs->controls[i].addr = addr; } /* no IQ_ESCR0/1 on some models, we save a seconde time BSU_ESCR0/1 * to avoid special case in nmi_{save|restore}_registers() */ if (boot_cpu_data.x86_model >= 0x3) { for (addr = MSR_P4_BSU_ESCR0 + stag; addr <= MSR_P4_BSU_ESCR1; ++i, addr += addr_increment()) { msrs->controls[i].addr = addr; } } else { for (addr = MSR_P4_IQ_ESCR0 + stag; addr <= MSR_P4_IQ_ESCR1; ++i, addr += addr_increment()) { msrs->controls[i].addr = addr; } } for (addr = MSR_P4_RAT_ESCR0 + stag; addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) { msrs->controls[i].addr = addr; } for (addr = MSR_P4_MS_ESCR0 + stag; addr <= MSR_P4_TC_ESCR1; ++i, addr += addr_increment()) { msrs->controls[i].addr = addr; } for (addr = MSR_P4_IX_ESCR0 + stag; addr <= MSR_P4_CRU_ESCR3; ++i, addr += addr_increment()) { msrs->controls[i].addr = addr; } /* there are 2 remaining non-contiguously located ESCRs */ if (num_counters == NUM_COUNTERS_NON_HT) { /* standard non-HT CPUs handle both remaining ESCRs*/ msrs->controls[i++].addr = MSR_P4_CRU_ESCR5; msrs->controls[i++].addr = MSR_P4_CRU_ESCR4; } else if (stag == 0) { /* HT CPUs give the first remainder to the even thread, as the 32nd control register */ msrs->controls[i++].addr = MSR_P4_CRU_ESCR4; } else { /* and two copies of the second to the odd thread, for the 22st and 23nd control registers */ msrs->controls[i++].addr = MSR_P4_CRU_ESCR5; msrs->controls[i++].addr = MSR_P4_CRU_ESCR5; } }
static int p4_fill_in_addresses(struct op_msrs * const msrs) { unsigned int i; unsigned int addr, cccraddr, stag; setup_num_counters(); stag = get_stagger(); /* the counter & cccr registers we pay attention to */ for (i = 0; i < num_counters; ++i) { addr = p4_counters[VIRT_CTR(stag, i)].counter_address; cccraddr = p4_counters[VIRT_CTR(stag, i)].cccr_address; if (reserve_perfctr_nmi(addr)) { msrs->counters[i].addr = addr; msrs->controls[i].addr = cccraddr; } } /* 43 ESCR registers in three or four discontiguous group */ for (addr = MSR_P4_BSU_ESCR0 + stag; addr < MSR_P4_IQ_ESCR0; ++i, addr += addr_increment()) { if (reserve_evntsel_nmi(addr)) msrs->controls[i].addr = addr; } /* no IQ_ESCR0/1 on some models, we save a seconde time BSU_ESCR0/1 * to avoid special case in nmi_{save|restore}_registers() */ if (boot_cpu_data.x86_model >= 0x3) { for (addr = MSR_P4_BSU_ESCR0 + stag; addr <= MSR_P4_BSU_ESCR1; ++i, addr += addr_increment()) { if (reserve_evntsel_nmi(addr)) msrs->controls[i].addr = addr; } } else { for (addr = MSR_P4_IQ_ESCR0 + stag; addr <= MSR_P4_IQ_ESCR1; ++i, addr += addr_increment()) { if (reserve_evntsel_nmi(addr)) msrs->controls[i].addr = addr; } } for (addr = MSR_P4_RAT_ESCR0 + stag; addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) { if (reserve_evntsel_nmi(addr)) msrs->controls[i].addr = addr; } for (addr = MSR_P4_MS_ESCR0 + stag; addr <= MSR_P4_TC_ESCR1; ++i, addr += addr_increment()) { if (reserve_evntsel_nmi(addr)) msrs->controls[i].addr = addr; } for (addr = MSR_P4_IX_ESCR0 + stag; addr <= MSR_P4_CRU_ESCR3; ++i, addr += addr_increment()) { if (reserve_evntsel_nmi(addr)) msrs->controls[i].addr = addr; } /* there are 2 remaining non-contiguously located ESCRs */ if (num_counters == NUM_COUNTERS_NON_HT) { /* standard non-HT CPUs handle both remaining ESCRs*/ if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR5)) msrs->controls[i++].addr = MSR_P4_CRU_ESCR5; if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR4)) msrs->controls[i++].addr = MSR_P4_CRU_ESCR4; } else if (stag == 0) { /* HT CPUs give the first remainder to the even thread, as the 32nd control register */ if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR4)) msrs->controls[i++].addr = MSR_P4_CRU_ESCR4; } else { /* and two copies of the second to the odd thread, for the 22st and 23nd control registers */ if (reserve_evntsel_nmi(MSR_P4_CRU_ESCR5)) { msrs->controls[i++].addr = MSR_P4_CRU_ESCR5; msrs->controls[i++].addr = MSR_P4_CRU_ESCR5; } } for (i = 0; i < num_counters; ++i) { if (!counter_config[i].enabled) continue; if (msrs->controls[i].addr) continue; op_x86_warn_reserved(i); p4_shutdown(msrs); return -EBUSY; } return 0; }