/**
* Allocates physical memory which satisfy the given constraints.
*
* @param uPhysHi The upper physical address limit (inclusive).
* @param puPhys Where to store the physical address of the allocated
* memory. Optional, can be NULL.
* @param cb Size of allocation.
* @param uAlignment Alignment.
* @param fContig Whether the memory must be physically contiguous or
* not.
*
* @returns Virtual address of allocated memory block or NULL if allocation
* failed.
*/
DECLHIDDEN(void *) rtR0SolMemAlloc(uint64_t uPhysHi, uint64_t *puPhys, size_t cb, uint64_t uAlignment, bool fContig)
{
if ((cb & PAGEOFFSET) != 0)
return NULL;
size_t cPages = (cb + PAGESIZE - 1) >> PAGESHIFT;
if (!cPages)
return NULL;
ddi_dma_attr_t DmaAttr = s_rtR0SolDmaAttr;
DmaAttr.dma_attr_addr_hi = uPhysHi;
DmaAttr.dma_attr_align = uAlignment;
if (!fContig)
DmaAttr.dma_attr_sgllen = cPages > INT_MAX ? INT_MAX - 1 : cPages;
else
AssertRelease(DmaAttr.dma_attr_sgllen == 1);
void *pvMem = contig_alloc(cb, &DmaAttr, PAGESIZE, 1 /* can sleep */);
if (!pvMem)
{
LogRel(("rtR0SolMemAlloc failed. cb=%u Align=%u fContig=%d\n", (unsigned)cb, (unsigned)uAlignment, fContig));
return NULL;
}
pfn_t PageFrameNum = hat_getpfnum(kas.a_hat, (caddr_t)pvMem);
AssertRelease(PageFrameNum != PFN_INVALID);
if (puPhys)
*puPhys = (uint64_t)PageFrameNum << PAGESHIFT;
return pvMem;
}
/*
* Reserve memory under PA 1G for mapping the new kernel and boot archive.
* This function is only called if fastreboot_onpanic is *not* set.
*/
static void
fastboot_reserve_mem(fastboot_info_t *nk)
{
int i;
/*
* A valid kernel is in place. No need to reserve any memory.
*/
if (nk->fi_valid)
return;
/*
* Reserve memory under PA 1G for PTE lists.
*/
for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) {
fastboot_file_t *fb = &nk->fi_files[i];
size_t fsize_roundup, size;
fsize_roundup = P2ROUNDUP_TYPED(saved_file_size[i],
PAGESIZE, size_t);
size = FASTBOOT_PTE_LIST_SIZE(fsize_roundup);
if ((fb->fb_pte_list_va = contig_alloc(size,
&fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) {
return;
}
fb->fb_pte_list_size = size;
}
/*
* Reserve memory under PA 1G for page tables.
*/
if ((nk->fi_pagetable_va =
(uintptr_t)contig_alloc(fastboot_pagetable_size,
&fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) {
return;
}
nk->fi_pagetable_size = fastboot_pagetable_size;
/*
* Reserve memory under PA 1G for multiboot structure.
*/
if ((nk->fi_new_mbi_va = (uintptr_t)contig_alloc(fastboot_mbi_size,
&fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) {
return;
}
nk->fi_mbi_size = fastboot_mbi_size;
}
/*
* This function performs the following tasks:
* - Read the sizes of the new kernel and boot archive.
* - Allocate memory for the new kernel and boot archive.
* - Allocate memory for page tables necessary for mapping the memory
* allocated for the files.
* - Read the new kernel and boot archive into memory.
* - Map in the fast reboot switcher.
* - Load the fast reboot switcher to FASTBOOT_SWTCH_PA.
* - Build the new multiboot_info structure
* - Build page tables for the low 1G of physical memory.
* - Mark the data structure as valid if all steps have succeeded.
*/
void
fastboot_load_kernel(char *mdep)
{
void *buf = NULL;
int i;
fastboot_file_t *fb;
uint32_t dboot_start_offset;
char kern_bootpath[OBP_MAXPATHLEN];
extern uintptr_t postbootkernelbase;
uintptr_t saved_kernelbase;
int bootpath_len = 0;
int is_failsafe = 0;
int is_retry = 0;
uint64_t end_addr;
if (!fastreboot_capable)
return;
if (newkernel.fi_valid)
fastboot_free_newkernel(&newkernel);
saved_kernelbase = postbootkernelbase;
postbootkernelbase = 0;
/*
* Initialize various HAT related fields in the data structure
*/
fastboot_init_fields(&newkernel);
bzero(kern_bootpath, OBP_MAXPATHLEN);
/*
* Process the boot argument
*/
bzero(fastboot_args, OBP_MAXPATHLEN);
fastboot_parse_mdep(mdep, kern_bootpath, &bootpath_len, fastboot_args);
/*
* Make sure we get the null character
*/
bcopy(kern_bootpath, fastboot_filename[FASTBOOT_NAME_UNIX],
bootpath_len);
bcopy(kern_bootfile,
&fastboot_filename[FASTBOOT_NAME_UNIX][bootpath_len],
strlen(kern_bootfile) + 1);
bcopy(kern_bootpath, fastboot_filename[FASTBOOT_NAME_BOOTARCHIVE],
bootpath_len);
if (bcmp(kern_bootfile, FAILSAFE_BOOTFILE32,
(sizeof (FAILSAFE_BOOTFILE32) - 1)) == 0 ||
bcmp(kern_bootfile, FAILSAFE_BOOTFILE64,
(sizeof (FAILSAFE_BOOTFILE64) - 1)) == 0) {
is_failsafe = 1;
}
load_kernel_retry:
/*
* Read in unix and boot_archive
*/
end_addr = DBOOT_ENTRY_ADDRESS;
for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) {
struct _buf *file;
uintptr_t va;
uint64_t fsize;
size_t fsize_roundup, pt_size;
int page_index;
uintptr_t offset;
ddi_dma_attr_t dma_attr = fastboot_dma_attr;
dprintf("fastboot_filename[%d] = %s\n",
i, fastboot_filename[i]);
if ((file = kobj_open_file(fastboot_filename[i])) ==
(struct _buf *)-1) {
cmn_err(CE_NOTE, "!Fastboot: Couldn't open %s",
fastboot_filename[i]);
goto err_out;
}
if (kobj_get_filesize(file, &fsize) != 0) {
cmn_err(CE_NOTE,
"!Fastboot: Couldn't get filesize for %s",
fastboot_filename[i]);
goto err_out;
}
fsize_roundup = P2ROUNDUP_TYPED(fsize, PAGESIZE, size_t);
/*
* Where the files end in physical memory after being
* relocated by the fast boot switcher.
*/
end_addr += fsize_roundup;
if (end_addr > fastboot_below_1G_dma_attr.dma_attr_addr_hi) {
cmn_err(CE_NOTE, "!Fastboot: boot archive is too big");
goto err_out;
}
/*
* Adjust dma_attr_addr_lo so that the new kernel and boot
* archive will not be overridden during relocation.
*/
if (end_addr > fastboot_dma_attr.dma_attr_addr_lo ||
end_addr > fastboot_below_1G_dma_attr.dma_attr_addr_lo) {
if (is_retry) {
/*
* If we have already tried and didn't succeed,
* just give up.
*/
cmn_err(CE_NOTE,
"!Fastboot: boot archive is too big");
goto err_out;
} else {
/* Set the flag so we don't keep retrying */
is_retry++;
/* Adjust dma_attr_addr_lo */
fastboot_dma_attr.dma_attr_addr_lo = end_addr;
fastboot_below_1G_dma_attr.dma_attr_addr_lo =
end_addr;
/*
* Free the memory we have already allocated
* whose physical addresses might not fit
* the new lo and hi constraints.
*/
fastboot_free_mem(&newkernel, end_addr);
goto load_kernel_retry;
}
}
if (!fastboot_contig)
dma_attr.dma_attr_sgllen = (fsize / PAGESIZE) +
(((fsize % PAGESIZE) == 0) ? 0 : 1);
if ((buf = contig_alloc(fsize, &dma_attr, PAGESIZE, 0))
== NULL) {
cmn_err(CE_NOTE, fastboot_enomem_msg, fsize, "64G");
goto err_out;
}
va = P2ROUNDUP_TYPED((uintptr_t)buf, PAGESIZE, uintptr_t);
if (kobj_read_file(file, (char *)va, fsize, 0) < 0) {
cmn_err(CE_NOTE, "!Fastboot: Couldn't read %s",
fastboot_filename[i]);
goto err_out;
}
fb = &newkernel.fi_files[i];
fb->fb_va = va;
fb->fb_size = fsize;
fb->fb_sectcnt = 0;
pt_size = FASTBOOT_PTE_LIST_SIZE(fsize_roundup);
/*
* If we have reserved memory but it not enough, free it.
*/
if (fb->fb_pte_list_size && fb->fb_pte_list_size < pt_size) {
contig_free((void *)fb->fb_pte_list_va,
fb->fb_pte_list_size);
fb->fb_pte_list_size = 0;
}
if (fb->fb_pte_list_size == 0) {
if ((fb->fb_pte_list_va =
(x86pte_t *)contig_alloc(pt_size,
&fastboot_below_1G_dma_attr, PAGESIZE, 0))
== NULL) {
cmn_err(CE_NOTE, fastboot_enomem_msg,
(uint64_t)pt_size, "1G");
goto err_out;
}
/*
* fb_pte_list_size must be set after the allocation
* succeeds as it's used to determine how much memory to
* free.
*/
fb->fb_pte_list_size = pt_size;
}
bzero((void *)(fb->fb_pte_list_va), fb->fb_pte_list_size);
fb->fb_pte_list_pa = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat,
(caddr_t)fb->fb_pte_list_va));
for (page_index = 0, offset = 0; offset < fb->fb_size;
offset += PAGESIZE) {
uint64_t paddr;
paddr = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat,
(caddr_t)fb->fb_va + offset));
ASSERT(paddr >= fastboot_dma_attr.dma_attr_addr_lo);
/*
* Include the pte_bits so we don't have to make
* it in assembly.
*/
fb->fb_pte_list_va[page_index++] = (x86pte_t)
(paddr | pte_bits);
}
fb->fb_pte_list_va[page_index] = FASTBOOT_TERMINATE;
if (i == FASTBOOT_UNIX) {
Ehdr *ehdr = (Ehdr *)va;
int j;
/*
* Sanity checks:
*/
for (j = 0; j < SELFMAG; j++) {
if (ehdr->e_ident[j] != ELFMAG[j]) {
cmn_err(CE_NOTE, "!Fastboot: Bad ELF "
"signature");
goto err_out;
}
}
if (ehdr->e_ident[EI_CLASS] == ELFCLASS32 &&
ehdr->e_ident[EI_DATA] == ELFDATA2LSB &&
ehdr->e_machine == EM_386) {
fb->fb_sectcnt = sizeof (fb->fb_sections) /
sizeof (fb->fb_sections[0]);
if (fastboot_elf32_find_loadables((void *)va,
fsize, &fb->fb_sections[0],
&fb->fb_sectcnt, &dboot_start_offset) < 0) {
cmn_err(CE_NOTE, "!Fastboot: ELF32 "
"program section failure");
goto err_out;
}
if (fb->fb_sectcnt == 0) {
cmn_err(CE_NOTE, "!Fastboot: No ELF32 "
"program sections found");
goto err_out;
}
if (is_failsafe) {
/* Failsafe boot_archive */
bcopy(BOOTARCHIVE32_FAILSAFE,
&fastboot_filename
[FASTBOOT_NAME_BOOTARCHIVE]
[bootpath_len],
sizeof (BOOTARCHIVE32_FAILSAFE));
} else {
bcopy(BOOTARCHIVE32,
&fastboot_filename
[FASTBOOT_NAME_BOOTARCHIVE]
[bootpath_len],
sizeof (BOOTARCHIVE32));
}
} else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64 &&
ehdr->e_ident[EI_DATA] == ELFDATA2LSB &&
ehdr->e_machine == EM_AMD64) {
if (fastboot_elf64_find_dboot_load_offset(
(void *)va, fsize, &dboot_start_offset)
!= 0) {
cmn_err(CE_NOTE, "!Fastboot: Couldn't "
"find ELF64 dboot entry offset");
goto err_out;
}
if (!is_x86_feature(x86_featureset,
X86FSET_64) ||
!is_x86_feature(x86_featureset,
X86FSET_PAE)) {
cmn_err(CE_NOTE, "Fastboot: Cannot "
"reboot to %s: "
"not a 64-bit capable system",
kern_bootfile);
goto err_out;
}
if (is_failsafe) {
/* Failsafe boot_archive */
bcopy(BOOTARCHIVE64_FAILSAFE,
&fastboot_filename
[FASTBOOT_NAME_BOOTARCHIVE]
[bootpath_len],
sizeof (BOOTARCHIVE64_FAILSAFE));
} else {
bcopy(BOOTARCHIVE64,
&fastboot_filename
[FASTBOOT_NAME_BOOTARCHIVE]
[bootpath_len],
sizeof (BOOTARCHIVE64));
}
} else {
cmn_err(CE_NOTE, "!Fastboot: Unknown ELF type");
goto err_out;
}
fb->fb_dest_pa = DBOOT_ENTRY_ADDRESS -
dboot_start_offset;
fb->fb_next_pa = DBOOT_ENTRY_ADDRESS + fsize_roundup;
} else {
fb->fb_dest_pa = newkernel.fi_files[i - 1].fb_next_pa;
fb->fb_next_pa = fb->fb_dest_pa + fsize_roundup;
}
kobj_close_file(file);
}
/*
* Add the function that will switch us to 32-bit protected mode
*/
fb = &newkernel.fi_files[FASTBOOT_SWTCH];
fb->fb_va = fb->fb_dest_pa = FASTBOOT_SWTCH_PA;
fb->fb_size = MMU_PAGESIZE;
hat_devload(kas.a_hat, (caddr_t)fb->fb_va,
MMU_PAGESIZE, mmu_btop(fb->fb_dest_pa),
PROT_READ | PROT_WRITE | PROT_EXEC,
HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
/*
* Build the new multiboot_info structure
*/
if (fastboot_build_mbi(fastboot_args, &newkernel) != 0) {
goto err_out;
}
/*
* Build page table for low 1G physical memory. Use big pages.
* Allocate 4 (5 for amd64) pages for the page tables.
* 1 page for PML4 (amd64)
* 1 page for Page-Directory-Pointer Table
* 2 pages for Page Directory
* 1 page for Page Table.
* The page table entry will be rewritten to map the physical
* address as we do the copying.
*/
if (newkernel.fi_has_pae) {
#ifdef __amd64
size_t size = MMU_PAGESIZE * 5;
#else
size_t size = MMU_PAGESIZE * 4;
#endif /* __amd64 */
if (newkernel.fi_pagetable_size && newkernel.fi_pagetable_size
< size) {
contig_free((void *)newkernel.fi_pagetable_va,
newkernel.fi_pagetable_size);
newkernel.fi_pagetable_size = 0;
}
if (newkernel.fi_pagetable_size == 0) {
if ((newkernel.fi_pagetable_va = (uintptr_t)
contig_alloc(size, &fastboot_below_1G_dma_attr,
MMU_PAGESIZE, 0)) == NULL) {
cmn_err(CE_NOTE, fastboot_enomem_msg,
(uint64_t)size, "1G");
goto err_out;
}
/*
* fi_pagetable_size must be set after the allocation
* succeeds as it's used to determine how much memory to
* free.
*/
newkernel.fi_pagetable_size = size;
}
bzero((void *)(newkernel.fi_pagetable_va), size);
newkernel.fi_pagetable_pa =
mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat,
(caddr_t)newkernel.fi_pagetable_va));
newkernel.fi_last_table_pa = newkernel.fi_pagetable_pa +
size - MMU_PAGESIZE;
newkernel.fi_next_table_va = newkernel.fi_pagetable_va +
MMU_PAGESIZE;
newkernel.fi_next_table_pa = newkernel.fi_pagetable_pa +
MMU_PAGESIZE;
fastboot_build_pagetables(&newkernel);
}
/* Generate MD5 checksums */
fastboot_cksum_generate(&newkernel);
/* Mark it as valid */
newkernel.fi_valid = 1;
newkernel.fi_magic = FASTBOOT_MAGIC;
postbootkernelbase = saved_kernelbase;
return;
err_out:
postbootkernelbase = saved_kernelbase;
newkernel.fi_valid = 0;
fastboot_free_newkernel(&newkernel);
}
/*
* Create multiboot info structure (mbi) base on the saved mbi.
* Recalculate values of the pointer type fields in the data
* structure based on the new starting physical address of the
* data structure.
*/
static int
fastboot_build_mbi(char *mdep, fastboot_info_t *nk)
{
mb_module_t *mbp;
multiboot_info_t *mbi; /* pointer to multiboot structure */
uintptr_t start_addr_va; /* starting VA of mbi */
uintptr_t start_addr_pa; /* starting PA of mbi */
size_t offs = 0; /* offset from the starting address */
size_t arglen; /* length of the command line arg */
size_t size; /* size of the memory reserved for mbi */
size_t mdnsz; /* length of the boot archive name */
/*
* If mdep is not NULL or empty, use the length of mdep + 1
* (for NULL terminating) as the length of the new command
* line; else use the saved command line length as the
* length for the new command line.
*/
if (mdep != NULL && strlen(mdep) != 0) {
arglen = strlen(mdep) + 1;
} else {
arglen = saved_cmdline_len;
}
/*
* Allocate memory for the new multiboot info structure (mbi).
* If we have reserved memory for mbi but it's not enough,
* free it and reallocate.
*/
size = PAGESIZE + P2ROUNDUP(arglen, PAGESIZE);
if (nk->fi_mbi_size && nk->fi_mbi_size < size) {
contig_free((void *)nk->fi_new_mbi_va, nk->fi_mbi_size);
nk->fi_mbi_size = 0;
}
if (nk->fi_mbi_size == 0) {
if ((nk->fi_new_mbi_va =
(uintptr_t)contig_alloc(size, &fastboot_below_1G_dma_attr,
PAGESIZE, 0)) == NULL) {
cmn_err(CE_NOTE, fastboot_enomem_msg,
(uint64_t)size, "1G");
return (-1);
}
/*
* fi_mbi_size must be set after the allocation succeeds
* as it's used to determine how much memory to free.
*/
nk->fi_mbi_size = size;
}
/*
* Initalize memory
*/
bzero((void *)nk->fi_new_mbi_va, nk->fi_mbi_size);
/*
* Get PA for the new mbi
*/
start_addr_va = nk->fi_new_mbi_va;
start_addr_pa = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat,
(caddr_t)start_addr_va));
nk->fi_new_mbi_pa = (paddr_t)start_addr_pa;
/*
* Populate the rest of the fields in the data structure
*/
/*
* Copy from the saved mbi to preserve all non-pointer type fields.
*/
mbi = (multiboot_info_t *)start_addr_va;
bcopy(&saved_mbi, mbi, sizeof (*mbi));
/*
* Recalculate mods_addr. Set mod_start and mod_end based on
* the physical address of the new boot archive. Set mod_name
* to the name of the new boto archive.
*/
offs += sizeof (multiboot_info_t);
mbi->mods_addr = start_addr_pa + offs;
mbp = (mb_module_t *)(start_addr_va + offs);
mbp->mod_start = nk->fi_files[FASTBOOT_BOOTARCHIVE].fb_dest_pa;
mbp->mod_end = nk->fi_files[FASTBOOT_BOOTARCHIVE].fb_next_pa;
offs += sizeof (mb_module_t);
mdnsz = strlen(fastboot_filename[FASTBOOT_NAME_BOOTARCHIVE]) + 1;
bcopy(fastboot_filename[FASTBOOT_NAME_BOOTARCHIVE],
(void *)(start_addr_va + offs), mdnsz);
mbp->mod_name = start_addr_pa + offs;
mbp->reserved = 0;
/*
* Make sure the offset is 16-byte aligned to avoid unaligned access.
*/
offs += mdnsz;
offs = P2ROUNDUP_TYPED(offs, 16, size_t);
/*
* Recalculate mmap_addr
*/
mbi->mmap_addr = start_addr_pa + offs;
bcopy((void *)(uintptr_t)saved_mmap, (void *)(start_addr_va + offs),
saved_mbi.mmap_length);
offs += saved_mbi.mmap_length;
/*
* Recalculate drives_addr
*/
mbi->drives_addr = start_addr_pa + offs;
bcopy((void *)(uintptr_t)saved_drives, (void *)(start_addr_va + offs),
saved_mbi.drives_length);
offs += saved_mbi.drives_length;
/*
* Recalculate the address of cmdline. Set cmdline to contain the
* new boot argument.
*/
mbi->cmdline = start_addr_pa + offs;
if (mdep != NULL && strlen(mdep) != 0) {
bcopy(mdep, (void *)(start_addr_va + offs), arglen);
} else {
bcopy((void *)saved_cmdline, (void *)(start_addr_va + offs),
arglen);
}
/* clear fields and flags that are not copied */
bzero(&mbi->config_table,
sizeof (*mbi) - offsetof(multiboot_info_t, config_table));
mbi->flags &= ~(MB_INFO_CONFIG_TABLE | MB_INFO_BOOT_LOADER_NAME |
MB_INFO_APM_TABLE | MB_INFO_VIDEO_INFO);
return (0);
}