Example #1
0
/**
 * kvm_mips_build_ret_to_host() - Assemble code to return to the host.
 * @addr:	Address to start writing code.
 *
 * Assemble the code to handle return from the guest exit handler
 * (kvm_mips_handle_exit()) back to the host, i.e. to the caller of the vcpu_run
 * function generated by kvm_mips_build_vcpu_run().
 *
 * Returns:	Next address after end of written function.
 */
static void *kvm_mips_build_ret_to_host(void *addr)
{
	u32 *p = addr;
	unsigned int i;

	/* EBASE is already pointing to Linux */
	UASM_i_LW(&p, K1, offsetof(struct kvm_vcpu_arch, host_stack), K1);
	uasm_i_addiu(&p, K1, K1, -(int)sizeof(struct pt_regs));

	/*
	 * r2/v0 is the return code, shift it down by 2 (arithmetic)
	 * to recover the err code
	 */
	uasm_i_sra(&p, K0, V0, 2);
	uasm_i_move(&p, V0, K0);

	/* Load context saved on the host stack */
	for (i = 16; i < 31; ++i) {
		if (i == 24)
			i = 28;
		UASM_i_LW(&p, i, offsetof(struct pt_regs, regs[i]), K1);
	}

	/* Restore RDHWR access */
	UASM_i_LA_mostly(&p, K0, (long)&hwrena);
	uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0);
	uasm_i_mtc0(&p, K0, C0_HWRENA);

	/* Restore RA, which is the address we will return to */
	UASM_i_LW(&p, RA, offsetof(struct pt_regs, regs[RA]), K1);
	uasm_i_jr(&p, RA);
	 uasm_i_nop(&p);

	return p;
}
Example #2
0
static inline void
pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off)
{
	if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) {
		if (off > 0x7fff) {
			uasm_i_lui(buf, T9, uasm_rel_hi(off));
			uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
		} else
			uasm_i_addiu(buf, T9, ZERO, off);
		uasm_i_daddu(buf, reg1, reg2, T9);
	} else {
		if (off > 0x7fff) {
			uasm_i_lui(buf, T9, uasm_rel_hi(off));
			uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
			UASM_i_ADDU(buf, reg1, reg2, T9);
		} else
			UASM_i_ADDIU(buf, reg1, reg2, off);
	}
}
Example #3
0
static void __init cps_gen_cache_routine(u32 **pp, struct uasm_label **pl,
					 struct uasm_reloc **pr,
					 const struct cache_desc *cache,
					 unsigned op, int lbl)
{
	unsigned cache_size = cache->ways << cache->waybit;
	unsigned i;
	const unsigned unroll_lines = 32;

	/* If the cache isn't present this function has it easy */
	if (cache->flags & MIPS_CACHE_NOT_PRESENT)
		return;

	/* Load base address */
	UASM_i_LA(pp, t0, (long)CKSEG0);

	/* Calculate end address */
	if (cache_size < 0x8000)
		uasm_i_addiu(pp, t1, t0, cache_size);
	else
		UASM_i_LA(pp, t1, (long)(CKSEG0 + cache_size));

	/* Start of cache op loop */
	uasm_build_label(pl, *pp, lbl);

	/* Generate the cache ops */
	for (i = 0; i < unroll_lines; i++) {
		if (cpu_has_mips_r6) {
			uasm_i_cache(pp, op, 0, t0);
			uasm_i_addiu(pp, t0, t0, cache->linesz);
		} else {
			uasm_i_cache(pp, op, i * cache->linesz, t0);
		}
	}

	if (!cpu_has_mips_r6)
		/* Update the base address */
		uasm_i_addiu(pp, t0, t0, unroll_lines * cache->linesz);

	/* Loop if we haven't reached the end address yet */
	uasm_il_bne(pp, pr, t0, t1, lbl);
	uasm_i_nop(pp);
}
Example #4
0
void UASM_i_LA(u32 **buf, unsigned int rs, long addr)
{
	UASM_i_LA_mostly(buf, rs, addr);
	if (uasm_rel_lo(addr)) {
		if (!uasm_in_compat_space_p(addr))
			uasm_i_daddiu(buf, rs, rs,
					uasm_rel_lo(addr));
		else
			uasm_i_addiu(buf, rs, rs,
					uasm_rel_lo(addr));
	}
}
Example #5
0
/**
 * kvm_mips_build_exit() - Assemble common guest exit handler.
 * @addr:	Address to start writing code.
 *
 * Assemble the generic guest exit handling code. This is called by the
 * exception vectors (generated by kvm_mips_build_exception()), and calls
 * kvm_mips_handle_exit(), then either resumes the guest or returns to the host
 * depending on the return value.
 *
 * Returns:	Next address after end of written function.
 */
void *kvm_mips_build_exit(void *addr)
{
	u32 *p = addr;
	unsigned int i;
	struct uasm_label labels[3];
	struct uasm_reloc relocs[3];
	struct uasm_label *l = labels;
	struct uasm_reloc *r = relocs;

	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));

	/*
	 * Generic Guest exception handler. We end up here when the guest
	 * does something that causes a trap to kernel mode.
	 *
	 * Both k0/k1 registers will have already been saved (k0 into the vcpu
	 * structure, and k1 into the scratch_tmp register).
	 *
	 * The k1 register will already contain the kvm_vcpu_arch pointer.
	 */

	/* Start saving Guest context to VCPU */
	for (i = 0; i < 32; ++i) {
		/* Guest k0/k1 saved later */
		if (i == K0 || i == K1)
			continue;
		UASM_i_SW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1);
	}

#ifndef CONFIG_CPU_MIPSR6
	/* We need to save hi/lo and restore them on the way out */
	uasm_i_mfhi(&p, T0);
	UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, hi), K1);

	uasm_i_mflo(&p, T0);
	UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, lo), K1);
#endif

	/* Finally save guest k1 to VCPU */
	uasm_i_ehb(&p);
	UASM_i_MFC0(&p, T0, scratch_tmp[0], scratch_tmp[1]);
	UASM_i_SW(&p, T0, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1);

	/* Now that context has been saved, we can use other registers */

	/* Restore vcpu */
	UASM_i_MFC0(&p, A1, scratch_vcpu[0], scratch_vcpu[1]);
	uasm_i_move(&p, S1, A1);

	/* Restore run (vcpu->run) */
	UASM_i_LW(&p, A0, offsetof(struct kvm_vcpu, run), A1);
	/* Save pointer to run in s0, will be saved by the compiler */
	uasm_i_move(&p, S0, A0);

	/*
	 * Save Host level EPC, BadVaddr and Cause to VCPU, useful to process
	 * the exception
	 */
	UASM_i_MFC0(&p, K0, C0_EPC);
	UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, pc), K1);

	UASM_i_MFC0(&p, K0, C0_BADVADDR);
	UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_badvaddr),
		  K1);

	uasm_i_mfc0(&p, K0, C0_CAUSE);
	uasm_i_sw(&p, K0, offsetof(struct kvm_vcpu_arch, host_cp0_cause), K1);

	/* Now restore the host state just enough to run the handlers */

	/* Switch EBASE to the one used by Linux */
	/* load up the host EBASE */
	uasm_i_mfc0(&p, V0, C0_STATUS);

	uasm_i_lui(&p, AT, ST0_BEV >> 16);
	uasm_i_or(&p, K0, V0, AT);

	uasm_i_mtc0(&p, K0, C0_STATUS);
	uasm_i_ehb(&p);

	UASM_i_LA_mostly(&p, K0, (long)&ebase);
	UASM_i_LW(&p, K0, uasm_rel_lo((long)&ebase), K0);
	build_set_exc_base(&p, K0);

	if (raw_cpu_has_fpu) {
		/*
		 * If FPU is enabled, save FCR31 and clear it so that later
		 * ctc1's don't trigger FPE for pending exceptions.
		 */
		uasm_i_lui(&p, AT, ST0_CU1 >> 16);
		uasm_i_and(&p, V1, V0, AT);
		uasm_il_beqz(&p, &r, V1, label_fpu_1);
		 uasm_i_nop(&p);
		uasm_i_cfc1(&p, T0, 31);
		uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.fcr31),
			  K1);
		uasm_i_ctc1(&p, ZERO, 31);
		uasm_l_fpu_1(&l, p);
	}

	if (cpu_has_msa) {
		/*
		 * If MSA is enabled, save MSACSR and clear it so that later
		 * instructions don't trigger MSAFPE for pending exceptions.
		 */
		uasm_i_mfc0(&p, T0, C0_CONFIG5);
		uasm_i_ext(&p, T0, T0, 27, 1); /* MIPS_CONF5_MSAEN */
		uasm_il_beqz(&p, &r, T0, label_msa_1);
		 uasm_i_nop(&p);
		uasm_i_cfcmsa(&p, T0, MSA_CSR);
		uasm_i_sw(&p, T0, offsetof(struct kvm_vcpu_arch, fpu.msacsr),
			  K1);
		uasm_i_ctcmsa(&p, MSA_CSR, ZERO);
		uasm_l_msa_1(&l, p);
	}

	/* Now that the new EBASE has been loaded, unset BEV and KSU_USER */
	uasm_i_addiu(&p, AT, ZERO, ~(ST0_EXL | KSU_USER | ST0_IE));
	uasm_i_and(&p, V0, V0, AT);
	uasm_i_lui(&p, AT, ST0_CU0 >> 16);
	uasm_i_or(&p, V0, V0, AT);
	uasm_i_mtc0(&p, V0, C0_STATUS);
	uasm_i_ehb(&p);

	/* Load up host GP */
	UASM_i_LW(&p, GP, offsetof(struct kvm_vcpu_arch, host_gp), K1);

	/* Need a stack before we can jump to "C" */
	UASM_i_LW(&p, SP, offsetof(struct kvm_vcpu_arch, host_stack), K1);

	/* Saved host state */
	UASM_i_ADDIU(&p, SP, SP, -(int)sizeof(struct pt_regs));

	/*
	 * XXXKYMA do we need to load the host ASID, maybe not because the
	 * kernel entries are marked GLOBAL, need to verify
	 */

	/* Restore host scratch registers, as we'll have clobbered them */
	kvm_mips_build_restore_scratch(&p, K0, SP);

	/* Restore RDHWR access */
	UASM_i_LA_mostly(&p, K0, (long)&hwrena);
	uasm_i_lw(&p, K0, uasm_rel_lo((long)&hwrena), K0);
	uasm_i_mtc0(&p, K0, C0_HWRENA);

	/* Jump to handler */
	/*
	 * XXXKYMA: not sure if this is safe, how large is the stack??
	 * Now jump to the kvm_mips_handle_exit() to see if we can deal
	 * with this in the kernel
	 */
	UASM_i_LA(&p, T9, (unsigned long)kvm_mips_handle_exit);
	uasm_i_jalr(&p, RA, T9);
	 UASM_i_ADDIU(&p, SP, SP, -CALLFRAME_SIZ);

	uasm_resolve_relocs(relocs, labels);

	p = kvm_mips_build_ret_from_exit(p);

	return p;
}
Example #6
0
/**
 * kvm_mips_build_enter_guest() - Assemble code to resume guest execution.
 * @addr:	Address to start writing code.
 *
 * Assemble the code to resume guest execution. This code is common between the
 * initial entry into the guest from the host, and returning from the exit
 * handler back to the guest.
 *
 * Returns:	Next address after end of written function.
 */
static void *kvm_mips_build_enter_guest(void *addr)
{
	u32 *p = addr;
	unsigned int i;
	struct uasm_label labels[2];
	struct uasm_reloc relocs[2];
	struct uasm_label *l = labels;
	struct uasm_reloc *r = relocs;

	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));

	/* Set Guest EPC */
	UASM_i_LW(&p, T0, offsetof(struct kvm_vcpu_arch, pc), K1);
	UASM_i_MTC0(&p, T0, C0_EPC);

	/* Set the ASID for the Guest Kernel */
	UASM_i_LW(&p, T0, offsetof(struct kvm_vcpu_arch, cop0), K1);
	UASM_i_LW(&p, T0, offsetof(struct mips_coproc, reg[MIPS_CP0_STATUS][0]),
		  T0);
	uasm_i_andi(&p, T0, T0, KSU_USER | ST0_ERL | ST0_EXL);
	uasm_i_xori(&p, T0, T0, KSU_USER);
	uasm_il_bnez(&p, &r, T0, label_kernel_asid);
	 UASM_i_ADDIU(&p, T1, K1,
		      offsetof(struct kvm_vcpu_arch, guest_kernel_asid));
	/* else user */
	UASM_i_ADDIU(&p, T1, K1,
		     offsetof(struct kvm_vcpu_arch, guest_user_asid));
	uasm_l_kernel_asid(&l, p);

	/* t1: contains the base of the ASID array, need to get the cpu id  */
	/* smp_processor_id */
	uasm_i_lw(&p, T2, offsetof(struct thread_info, cpu), GP);
	/* x4 */
	uasm_i_sll(&p, T2, T2, 2);
	UASM_i_ADDU(&p, T3, T1, T2);
	uasm_i_lw(&p, K0, 0, T3);
#ifdef CONFIG_MIPS_ASID_BITS_VARIABLE
	/* x sizeof(struct cpuinfo_mips)/4 */
	uasm_i_addiu(&p, T3, ZERO, sizeof(struct cpuinfo_mips)/4);
	uasm_i_mul(&p, T2, T2, T3);

	UASM_i_LA_mostly(&p, AT, (long)&cpu_data[0].asid_mask);
	UASM_i_ADDU(&p, AT, AT, T2);
	UASM_i_LW(&p, T2, uasm_rel_lo((long)&cpu_data[0].asid_mask), AT);
	uasm_i_and(&p, K0, K0, T2);
#else
	uasm_i_andi(&p, K0, K0, MIPS_ENTRYHI_ASID);
#endif
	uasm_i_mtc0(&p, K0, C0_ENTRYHI);
	uasm_i_ehb(&p);

	/* Disable RDHWR access */
	uasm_i_mtc0(&p, ZERO, C0_HWRENA);

	/* load the guest context from VCPU and return */
	for (i = 1; i < 32; ++i) {
		/* Guest k0/k1 loaded later */
		if (i == K0 || i == K1)
			continue;
		UASM_i_LW(&p, i, offsetof(struct kvm_vcpu_arch, gprs[i]), K1);
	}

#ifndef CONFIG_CPU_MIPSR6
	/* Restore hi/lo */
	UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, hi), K1);
	uasm_i_mthi(&p, K0);

	UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, lo), K1);
	uasm_i_mtlo(&p, K0);
#endif

	/* Restore the guest's k0/k1 registers */
	UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, gprs[K0]), K1);
	UASM_i_LW(&p, K1, offsetof(struct kvm_vcpu_arch, gprs[K1]), K1);

	/* Jump to guest */
	uasm_i_eret(&p);

	uasm_resolve_relocs(relocs, labels);

	return p;
}
Example #7
0
/**
 * kvm_mips_build_vcpu_run() - Assemble function to start running a guest VCPU.
 * @addr:	Address to start writing code.
 *
 * Assemble the start of the vcpu_run function to run a guest VCPU. The function
 * conforms to the following prototype:
 *
 * int vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu);
 *
 * The exit from the guest and return to the caller is handled by the code
 * generated by kvm_mips_build_ret_to_host().
 *
 * Returns:	Next address after end of written function.
 */
void *kvm_mips_build_vcpu_run(void *addr)
{
	u32 *p = addr;
	unsigned int i;

	/*
	 * A0: run
	 * A1: vcpu
	 */

	/* k0/k1 not being used in host kernel context */
	UASM_i_ADDIU(&p, K1, SP, -(int)sizeof(struct pt_regs));
	for (i = 16; i < 32; ++i) {
		if (i == 24)
			i = 28;
		UASM_i_SW(&p, i, offsetof(struct pt_regs, regs[i]), K1);
	}

	/* Save host status */
	uasm_i_mfc0(&p, V0, C0_STATUS);
	UASM_i_SW(&p, V0, offsetof(struct pt_regs, cp0_status), K1);

	/* Save scratch registers, will be used to store pointer to vcpu etc */
	kvm_mips_build_save_scratch(&p, V1, K1);

	/* VCPU scratch register has pointer to vcpu */
	UASM_i_MTC0(&p, A1, scratch_vcpu[0], scratch_vcpu[1]);

	/* Offset into vcpu->arch */
	UASM_i_ADDIU(&p, K1, A1, offsetof(struct kvm_vcpu, arch));

	/*
	 * Save the host stack to VCPU, used for exception processing
	 * when we exit from the Guest
	 */
	UASM_i_SW(&p, SP, offsetof(struct kvm_vcpu_arch, host_stack), K1);

	/* Save the kernel gp as well */
	UASM_i_SW(&p, GP, offsetof(struct kvm_vcpu_arch, host_gp), K1);

	/*
	 * Setup status register for running the guest in UM, interrupts
	 * are disabled
	 */
	UASM_i_LA(&p, K0, ST0_EXL | KSU_USER | ST0_BEV | ST0_KX_IF_64);
	uasm_i_mtc0(&p, K0, C0_STATUS);
	uasm_i_ehb(&p);

	/* load up the new EBASE */
	UASM_i_LW(&p, K0, offsetof(struct kvm_vcpu_arch, guest_ebase), K1);
	build_set_exc_base(&p, K0);

	/*
	 * Now that the new EBASE has been loaded, unset BEV, set
	 * interrupt mask as it was but make sure that timer interrupts
	 * are enabled
	 */
	uasm_i_addiu(&p, K0, ZERO, ST0_EXL | KSU_USER | ST0_IE | ST0_KX_IF_64);
	uasm_i_andi(&p, V0, V0, ST0_IM);
	uasm_i_or(&p, K0, K0, V0);
	uasm_i_mtc0(&p, K0, C0_STATUS);
	uasm_i_ehb(&p);

	p = kvm_mips_build_enter_guest(p);

	return p;
}
Example #8
0
static void __init install_trampoline(u32 *tramp, unsigned int sigreturn)
{
	uasm_i_addiu(&tramp, 2, 0, sigreturn);	
	uasm_i_syscall(&tramp, 0);
}
Example #9
0
static void * __init cps_gen_entry_code(unsigned cpu, enum cps_pm_state state)
{
	struct uasm_label *l = labels;
	struct uasm_reloc *r = relocs;
	u32 *buf, *p;
	const unsigned r_online = a0;
	const unsigned r_nc_count = a1;
	const unsigned r_pcohctl = t7;
	const unsigned max_instrs = 256;
	unsigned cpc_cmd;
	enum {
		lbl_incready = 1,
		lbl_poll_cont,
		lbl_secondary_hang,
		lbl_disable_coherence,
		lbl_flush_fsb,
		lbl_invicache,
		lbl_flushdcache,
		lbl_hang,
		lbl_set_cont,
		lbl_secondary_cont,
		lbl_decready,
	};

	/* Allocate a buffer to hold the generated code */
	p = buf = kcalloc(max_instrs, sizeof(u32), GFP_KERNEL);
	if (!buf)
		return NULL;

	/* Clear labels & relocs ready for (re)use */
	memset(labels, 0, sizeof(labels));
	memset(relocs, 0, sizeof(relocs));

	if (state == CPS_PM_POWER_GATED) {
		/* Power gating relies upon CPS SMP */
		if (!mips_cps_smp_in_use())
			goto out_err;

		/*
		 * Save CPU state. Note the non-standard calling convention
		 * with the return address placed in v0 to avoid clobbering
		 * the ra register before it is saved.
		 */
		UASM_i_LA(&p, t0, (long)mips_cps_pm_save);
		uasm_i_jalr(&p, v0, t0);
		uasm_i_nop(&p);
	}

	/*
	 * Load addresses of required CM & CPC registers. This is done early
	 * because they're needed in both the enable & disable coherence steps
	 * but in the coupled case the enable step will only run on one VPE.
	 */
	UASM_i_LA(&p, r_pcohctl, (long)_gcmp_base + GCMPCLCBOFS(COHCTL));

	if (coupled_coherence) {
		/* Increment ready_count */
		uasm_i_sync(&p, stype_ordering);
		uasm_build_label(&l, p, lbl_incready);
		uasm_i_ll(&p, t1, 0, r_nc_count);
		uasm_i_addiu(&p, t2, t1, 1);
		uasm_i_sc(&p, t2, 0, r_nc_count);
		uasm_il_beqz(&p, &r, t2, lbl_incready);
		uasm_i_addiu(&p, t1, t1, 1);

		/* Ordering barrier */
		uasm_i_sync(&p, stype_ordering);

		/*
		 * If this is the last VPE to become ready for non-coherence
		 * then it should branch below.
		 */
		uasm_il_beq(&p, &r, t1, r_online, lbl_disable_coherence);
		uasm_i_nop(&p);

		if (state < CPS_PM_POWER_GATED) {
			/*
			 * Otherwise this is not the last VPE to become ready
			 * for non-coherence. It needs to wait until coherence
			 * has been disabled before proceeding, which it will do
			 * by polling for the top bit of ready_count being set.
			 */
			uasm_i_addiu(&p, t1, zero, -1);
			uasm_build_label(&l, p, lbl_poll_cont);
			uasm_i_lw(&p, t0, 0, r_nc_count);
			uasm_il_bltz(&p, &r, t0, lbl_secondary_cont);
			uasm_i_ehb(&p);
			uasm_i_yield(&p, zero, t1);
			uasm_il_b(&p, &r, lbl_poll_cont);
			uasm_i_nop(&p);
		} else {
			/*
			 * The core will lose power & this VPE will not continue
			 * so it can simply halt here.
			 */
			uasm_i_addiu(&p, t0, zero, TCHALT_H);
			uasm_i_mtc0(&p, t0, 2, 4);
			uasm_build_label(&l, p, lbl_secondary_hang);
			uasm_il_b(&p, &r, lbl_secondary_hang);
			uasm_i_nop(&p);
		}
	}

	/*
	 * This is the point of no return - this VPE will now proceed to
	 * disable coherence. At this point we *must* be sure that no other
	 * VPE within the core will interfere with the L1 dcache.
	 */
	uasm_build_label(&l, p, lbl_disable_coherence);

	/* Invalidate the L1 icache */
	cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].icache,
			      Index_Invalidate_I, lbl_invicache);

	/* Writeback & invalidate the L1 dcache */
	cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].dcache,
			      Index_Writeback_Inv_D, lbl_flushdcache);

	/* Completion barrier */
	uasm_i_sync(&p, stype_memory);
	uasm_i_ehb(&p);

	/*
	 * Disable all but self interventions. The load from COHCTL is defined
	 * by the interAptiv & proAptiv SUMs as ensuring that the operation
	 * resulting from the preceeding store is complete.
	 */
	uasm_i_addiu(&p, t0, zero, 1 << cpu_data[cpu].core);
	uasm_i_sw(&p, t0, 0, r_pcohctl);
	uasm_i_lw(&p, t0, 0, r_pcohctl);

	/* Sync to ensure previous interventions are complete */
	uasm_i_sync(&p, stype_intervention);
	uasm_i_ehb(&p);

	/* Disable coherence */
	uasm_i_sw(&p, zero, 0, r_pcohctl);
	uasm_i_lw(&p, t0, 0, r_pcohctl);

	if (state >= CPS_PM_CLOCK_GATED) {
		/* TODO: determine whether required based on CPC version */
		cps_gen_flush_fsb(&p, &l, &r, &cpu_data[cpu].dcache,
				  lbl_flush_fsb);

		/* Determine the CPC command to issue */
		switch (state) {
		case CPS_PM_CLOCK_GATED:
			cpc_cmd = CPC_Cx_CMD_CLOCKOFF;
			break;
		case CPS_PM_POWER_GATED:
			cpc_cmd = CPC_Cx_CMD_PWRDOWN;
			break;
		default:
			BUG();
			goto out_err;
		}

		/* Issue the CPC command */
		UASM_i_LA(&p, t0, (long)addr_cpc_cl_cmd());
		uasm_i_addiu(&p, t1, zero, cpc_cmd);
		uasm_i_sw(&p, t1, 0, t0);

		if (state == CPS_PM_POWER_GATED) {
			/* If anything goes wrong just hang */
			uasm_build_label(&l, p, lbl_hang);
			uasm_il_b(&p, &r, lbl_hang);
			uasm_i_nop(&p);

			/*
			 * There's no point generating more code, the core is
			 * powered down & if powered back up will run from the
			 * reset vector not from here.
			 */
			goto gen_done;
		}

		/* Completion barrier */
		uasm_i_sync(&p, stype_memory);
		uasm_i_ehb(&p);
	}

	if (state == CPS_PM_NC_WAIT) {
		/*
		 * At this point it is safe for all VPEs to proceed with
		 * execution. This VPE will set the top bit of ready_count
		 * to indicate to the other VPEs that they may continue.
		 */
		if (coupled_coherence)
			cps_gen_set_top_bit(&p, &l, &r, r_nc_count,
					    lbl_set_cont);

		/*
		 * VPEs which did not disable coherence will continue
		 * executing, after coherence has been disabled, from this
		 * point.
		 */
		uasm_build_label(&l, p, lbl_secondary_cont);

		/* Now perform our wait */
		uasm_i_wait(&p, 0);
	}

	/*
	 * Re-enable coherence. Note that for CPS_PM_NC_WAIT all coupled VPEs
	 * will run this. The first will actually re-enable coherence & the
	 * rest will just be performing a rather unusual nop.
	 */
	uasm_i_addiu(&p, t0, zero, GCMP_CCB_COHCTL_DOMAIN_MSK);
	uasm_i_sw(&p, t0, 0, r_pcohctl);
	uasm_i_lw(&p, t0, 0, r_pcohctl);

	/* Completion barrier */
	uasm_i_sync(&p, stype_memory);
	uasm_i_ehb(&p);

	if (coupled_coherence && (state == CPS_PM_NC_WAIT)) {
		/* Decrement ready_count */
		uasm_build_label(&l, p, lbl_decready);
		uasm_i_sync(&p, stype_ordering);
		uasm_i_ll(&p, t1, 0, r_nc_count);
		uasm_i_addiu(&p, t2, t1, -1);
		uasm_i_sc(&p, t2, 0, r_nc_count);
		uasm_il_beqz(&p, &r, t2, lbl_decready);
		uasm_i_andi(&p, v0, t1, (1 << fls(smp_num_siblings)) - 1);

		/* Ordering barrier */
		uasm_i_sync(&p, stype_ordering);
	}

	if (coupled_coherence && (state == CPS_PM_CLOCK_GATED)) {
		/*
		 * At this point it is safe for all VPEs to proceed with
		 * execution. This VPE will set the top bit of ready_count
		 * to indicate to the other VPEs that they may continue.
		 */
		cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont);

		/*
		 * This core will be reliant upon another core sending a
		 * power-up command to the CPC in order to resume operation.
		 * Thus an arbitrary VPE can't trigger the core leaving the
		 * idle state and the one that disables coherence might as well
		 * be the one to re-enable it. The rest will continue from here
		 * after that has been done.
		 */
		uasm_build_label(&l, p, lbl_secondary_cont);

		/* Ordering barrier */
		uasm_i_sync(&p, stype_ordering);
	}

	/* The core is coherent, time to return to C code */
	uasm_i_jr(&p, ra);
	uasm_i_nop(&p);

gen_done:
	/* Ensure the code didn't exceed the resources allocated for it */
	BUG_ON((p - buf) > max_instrs);
	BUG_ON((l - labels) > ARRAY_SIZE(labels));
	BUG_ON((r - relocs) > ARRAY_SIZE(relocs));

	/* Patch branch offsets */
	uasm_resolve_relocs(relocs, labels);

	/* Flush the icache */
	local_flush_icache_range((unsigned long)buf, (unsigned long)p);

	return buf;
out_err:
	kfree(buf);
	return NULL;
}
Example #10
0
static void __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl,
				     struct uasm_reloc **pr,
				     const struct cache_desc *dcache,
				     int lbl)
{
	unsigned i, fsb_size = 8;
	unsigned num_loads = (fsb_size * 3) / 2;
	unsigned line_stride = 2;

	/*
	 * Ensure that the fill/store buffer (FSB) is not holding the results
	 * of a prefetch, since if it is then the CPC sequencer may become
	 * stuck in the D3 (ClrBus) state whilst entering a low power state.
	 */

	/* TODO: this is interAptiv-specific, generalise it */

	/* Preserve perf counter 1 setup */
	uasm_i_mfc0(pp, t2, 25, 2); /* PerfCtl1 */
	uasm_i_mfc0(pp, t3, 25, 3); /* PerfCnt1 */

	/* Setup perf counter 1 to count FSB full pipeline stalls */
	uasm_i_addiu(pp, t0, zero, 0x66f);
	uasm_i_mtc0(pp, t0, 25, 2); /* PerfCtl1 */
	uasm_i_ehb(pp);
	uasm_i_mtc0(pp, zero, 25, 3); /* PerfCnt1 */
	uasm_i_ehb(pp);

	/* Base address for loads */
	UASM_i_LA(pp, t0, (long)CKSEG0);

	/* Start of clear loop */
	uasm_build_label(pl, *pp, lbl);

	/* Perform some loads to fill the FSB */
	for (i = 0; i < num_loads; i++)
		uasm_i_lw(pp, zero, i * dcache->linesz * line_stride, t0);

	/*
	 * Invalidate the new D-cache entries so that the cache will need
	 * refilling (via the FSB) if the loop is executed again.
	 */
	for (i = 0; i < num_loads; i++) {
		uasm_i_cache(pp, Hit_Invalidate_D,
			     i * dcache->linesz * line_stride, t0);
		uasm_i_cache(pp, Hit_Writeback_Inv_SD,
			     i * dcache->linesz * line_stride, t0);
	}

	/* Completion barrier */
	uasm_i_sync(pp, stype_memory);
	uasm_i_ehb(pp);

	/* Check whether the pipeline stalled due to the FSB being full */
	uasm_i_mfc0(pp, t1, 25, 3); /* PerfCnt1 */

	/* Loop if it didn't */
	uasm_il_beqz(pp, pr, t1, lbl);
	uasm_i_nop(pp);

	/* Restore perf counter 1. The count may well now be wrong... */
	uasm_i_mtc0(pp, t2, 25, 2); /* PerfCtl1 */
	uasm_i_ehb(pp);
	uasm_i_mtc0(pp, t3, 25, 3); /* PerfCnt1 */
	uasm_i_ehb(pp);
}
Example #11
0
/*
 * BVADDR is the faulting address, PTR is scratch.
 * PTR will hold the pgd for vmalloc.
 */
static __init void
build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
			unsigned int bvaddr, unsigned int ptr)
{
	long swpd = (long)swapper_pg_dir;

#ifdef MODULE_START
	long modd = (long)module_pg_dir;

	uasm_l_module_alloc(l, *p);
	/*
	 * Assumption:
	 * VMALLOC_START >= 0xc000000000000000UL
	 * MODULE_START >= 0xe000000000000000UL
	 */
	UASM_i_SLL(p, ptr, bvaddr, 2);
	uasm_il_bgez(p, r, ptr, label_vmalloc);

	if (uasm_in_compat_space_p(MODULE_START) &&
	    !uasm_rel_lo(MODULE_START)) {
		uasm_i_lui(p, ptr, uasm_rel_hi(MODULE_START)); /* delay slot */
	} else {
		/* unlikely configuration */
		uasm_i_nop(p); /* delay slot */
		UASM_i_LA(p, ptr, MODULE_START);
	}
	uasm_i_dsubu(p, bvaddr, bvaddr, ptr);

	if (uasm_in_compat_space_p(modd) && !uasm_rel_lo(modd)) {
		uasm_il_b(p, r, label_vmalloc_done);
		uasm_i_lui(p, ptr, uasm_rel_hi(modd));
	} else {
		UASM_i_LA_mostly(p, ptr, modd);
		uasm_il_b(p, r, label_vmalloc_done);
		if (uasm_in_compat_space_p(modd))
			uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(modd));
		else
			uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(modd));
	}

	uasm_l_vmalloc(l, *p);
	if (uasm_in_compat_space_p(MODULE_START) &&
	    !uasm_rel_lo(MODULE_START) &&
	    MODULE_START << 32 == VMALLOC_START)
		uasm_i_dsll32(p, ptr, ptr, 0);	/* typical case */
	else
		UASM_i_LA(p, ptr, VMALLOC_START);
#else
	uasm_l_vmalloc(l, *p);
	UASM_i_LA(p, ptr, VMALLOC_START);
#endif
	uasm_i_dsubu(p, bvaddr, bvaddr, ptr);

	if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
		uasm_il_b(p, r, label_vmalloc_done);
		uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
	} else {
		UASM_i_LA_mostly(p, ptr, swpd);
		uasm_il_b(p, r, label_vmalloc_done);
		if (uasm_in_compat_space_p(swpd))
			uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
		else
			uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
	}
}
Example #12
0
static int __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl,
				    struct uasm_reloc **pr,
				    const struct cpuinfo_mips *cpu_info,
				    int lbl)
{
	unsigned i, fsb_size = 8;
	unsigned num_loads = (fsb_size * 3) / 2;
	unsigned line_stride = 2;
	unsigned line_size = cpu_info->dcache.linesz;
	unsigned perf_counter, perf_event;
	unsigned revision = cpu_info->processor_id & PRID_REV_MASK;

	/*
	 * Determine whether this CPU requires an FSB flush, and if so which
	 * performance counter/event reflect stalls due to a full FSB.
	 */
	switch (__get_cpu_type(cpu_info->cputype)) {
	case CPU_INTERAPTIV:
		perf_counter = 1;
		perf_event = 51;
		break;

	case CPU_PROAPTIV:
		/* Newer proAptiv cores don't require this workaround */
		if (revision >= PRID_REV_ENCODE_332(1, 1, 0))
			return 0;

		/* On older ones it's unavailable */
		return -1;

	/* CPUs which do not require the workaround */
	case CPU_P5600:
	case CPU_I6400:
		return 0;

	default:
		WARN_ONCE(1, "pm-cps: FSB flush unsupported for this CPU\n");
		return -1;
	}

	/*
	 * Ensure that the fill/store buffer (FSB) is not holding the results
	 * of a prefetch, since if it is then the CPC sequencer may become
	 * stuck in the D3 (ClrBus) state whilst entering a low power state.
	 */

	/* Preserve perf counter setup */
	uasm_i_mfc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */
	uasm_i_mfc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */

	/* Setup perf counter to count FSB full pipeline stalls */
	uasm_i_addiu(pp, t0, zero, (perf_event << 5) | 0xf);
	uasm_i_mtc0(pp, t0, 25, (perf_counter * 2) + 0); /* PerfCtlN */
	uasm_i_ehb(pp);
	uasm_i_mtc0(pp, zero, 25, (perf_counter * 2) + 1); /* PerfCntN */
	uasm_i_ehb(pp);

	/* Base address for loads */
	UASM_i_LA(pp, t0, (long)CKSEG0);

	/* Start of clear loop */
	uasm_build_label(pl, *pp, lbl);

	/* Perform some loads to fill the FSB */
	for (i = 0; i < num_loads; i++)
		uasm_i_lw(pp, zero, i * line_size * line_stride, t0);

	/*
	 * Invalidate the new D-cache entries so that the cache will need
	 * refilling (via the FSB) if the loop is executed again.
	 */
	for (i = 0; i < num_loads; i++) {
		uasm_i_cache(pp, Hit_Invalidate_D,
			     i * line_size * line_stride, t0);
		uasm_i_cache(pp, Hit_Writeback_Inv_SD,
			     i * line_size * line_stride, t0);
	}

	/* Completion barrier */
	uasm_i_sync(pp, stype_memory);
	uasm_i_ehb(pp);

	/* Check whether the pipeline stalled due to the FSB being full */
	uasm_i_mfc0(pp, t1, 25, (perf_counter * 2) + 1); /* PerfCntN */

	/* Loop if it didn't */
	uasm_il_beqz(pp, pr, t1, lbl);
	uasm_i_nop(pp);

	/* Restore perf counter 1. The count may well now be wrong... */
	uasm_i_mtc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */
	uasm_i_ehb(pp);
	uasm_i_mtc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */
	uasm_i_ehb(pp);

	return 0;
}