/** * Allocates physical memory which satisfy the given constraints. * * @param uPhysHi The upper physical address limit (inclusive). * @param puPhys Where to store the physical address of the allocated * memory. Optional, can be NULL. * @param cb Size of allocation. * @param uAlignment Alignment. * @param fContig Whether the memory must be physically contiguous or * not. * * @returns Virtual address of allocated memory block or NULL if allocation * failed. */ DECLHIDDEN(void *) rtR0SolMemAlloc(uint64_t uPhysHi, uint64_t *puPhys, size_t cb, uint64_t uAlignment, bool fContig) { if ((cb & PAGEOFFSET) != 0) return NULL; size_t cPages = (cb + PAGESIZE - 1) >> PAGESHIFT; if (!cPages) return NULL; ddi_dma_attr_t DmaAttr = s_rtR0SolDmaAttr; DmaAttr.dma_attr_addr_hi = uPhysHi; DmaAttr.dma_attr_align = uAlignment; if (!fContig) DmaAttr.dma_attr_sgllen = cPages > INT_MAX ? INT_MAX - 1 : cPages; else AssertRelease(DmaAttr.dma_attr_sgllen == 1); void *pvMem = contig_alloc(cb, &DmaAttr, PAGESIZE, 1 /* can sleep */); if (!pvMem) { LogRel(("rtR0SolMemAlloc failed. cb=%u Align=%u fContig=%d\n", (unsigned)cb, (unsigned)uAlignment, fContig)); return NULL; } pfn_t PageFrameNum = hat_getpfnum(kas.a_hat, (caddr_t)pvMem); AssertRelease(PageFrameNum != PFN_INVALID); if (puPhys) *puPhys = (uint64_t)PageFrameNum << PAGESHIFT; return pvMem; }
/* * Reserve memory under PA 1G for mapping the new kernel and boot archive. * This function is only called if fastreboot_onpanic is *not* set. */ static void fastboot_reserve_mem(fastboot_info_t *nk) { int i; /* * A valid kernel is in place. No need to reserve any memory. */ if (nk->fi_valid) return; /* * Reserve memory under PA 1G for PTE lists. */ for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { fastboot_file_t *fb = &nk->fi_files[i]; size_t fsize_roundup, size; fsize_roundup = P2ROUNDUP_TYPED(saved_file_size[i], PAGESIZE, size_t); size = FASTBOOT_PTE_LIST_SIZE(fsize_roundup); if ((fb->fb_pte_list_va = contig_alloc(size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) { return; } fb->fb_pte_list_size = size; } /* * Reserve memory under PA 1G for page tables. */ if ((nk->fi_pagetable_va = (uintptr_t)contig_alloc(fastboot_pagetable_size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) { return; } nk->fi_pagetable_size = fastboot_pagetable_size; /* * Reserve memory under PA 1G for multiboot structure. */ if ((nk->fi_new_mbi_va = (uintptr_t)contig_alloc(fastboot_mbi_size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) { return; } nk->fi_mbi_size = fastboot_mbi_size; }
/* * This function performs the following tasks: * - Read the sizes of the new kernel and boot archive. * - Allocate memory for the new kernel and boot archive. * - Allocate memory for page tables necessary for mapping the memory * allocated for the files. * - Read the new kernel and boot archive into memory. * - Map in the fast reboot switcher. * - Load the fast reboot switcher to FASTBOOT_SWTCH_PA. * - Build the new multiboot_info structure * - Build page tables for the low 1G of physical memory. * - Mark the data structure as valid if all steps have succeeded. */ void fastboot_load_kernel(char *mdep) { void *buf = NULL; int i; fastboot_file_t *fb; uint32_t dboot_start_offset; char kern_bootpath[OBP_MAXPATHLEN]; extern uintptr_t postbootkernelbase; uintptr_t saved_kernelbase; int bootpath_len = 0; int is_failsafe = 0; int is_retry = 0; uint64_t end_addr; if (!fastreboot_capable) return; if (newkernel.fi_valid) fastboot_free_newkernel(&newkernel); saved_kernelbase = postbootkernelbase; postbootkernelbase = 0; /* * Initialize various HAT related fields in the data structure */ fastboot_init_fields(&newkernel); bzero(kern_bootpath, OBP_MAXPATHLEN); /* * Process the boot argument */ bzero(fastboot_args, OBP_MAXPATHLEN); fastboot_parse_mdep(mdep, kern_bootpath, &bootpath_len, fastboot_args); /* * Make sure we get the null character */ bcopy(kern_bootpath, fastboot_filename[FASTBOOT_NAME_UNIX], bootpath_len); bcopy(kern_bootfile, &fastboot_filename[FASTBOOT_NAME_UNIX][bootpath_len], strlen(kern_bootfile) + 1); bcopy(kern_bootpath, fastboot_filename[FASTBOOT_NAME_BOOTARCHIVE], bootpath_len); if (bcmp(kern_bootfile, FAILSAFE_BOOTFILE32, (sizeof (FAILSAFE_BOOTFILE32) - 1)) == 0 || bcmp(kern_bootfile, FAILSAFE_BOOTFILE64, (sizeof (FAILSAFE_BOOTFILE64) - 1)) == 0) { is_failsafe = 1; } load_kernel_retry: /* * Read in unix and boot_archive */ end_addr = DBOOT_ENTRY_ADDRESS; for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { struct _buf *file; uintptr_t va; uint64_t fsize; size_t fsize_roundup, pt_size; int page_index; uintptr_t offset; ddi_dma_attr_t dma_attr = fastboot_dma_attr; dprintf("fastboot_filename[%d] = %s\n", i, fastboot_filename[i]); if ((file = kobj_open_file(fastboot_filename[i])) == (struct _buf *)-1) { cmn_err(CE_NOTE, "!Fastboot: Couldn't open %s", fastboot_filename[i]); goto err_out; } if (kobj_get_filesize(file, &fsize) != 0) { cmn_err(CE_NOTE, "!Fastboot: Couldn't get filesize for %s", fastboot_filename[i]); goto err_out; } fsize_roundup = P2ROUNDUP_TYPED(fsize, PAGESIZE, size_t); /* * Where the files end in physical memory after being * relocated by the fast boot switcher. */ end_addr += fsize_roundup; if (end_addr > fastboot_below_1G_dma_attr.dma_attr_addr_hi) { cmn_err(CE_NOTE, "!Fastboot: boot archive is too big"); goto err_out; } /* * Adjust dma_attr_addr_lo so that the new kernel and boot * archive will not be overridden during relocation. */ if (end_addr > fastboot_dma_attr.dma_attr_addr_lo || end_addr > fastboot_below_1G_dma_attr.dma_attr_addr_lo) { if (is_retry) { /* * If we have already tried and didn't succeed, * just give up. */ cmn_err(CE_NOTE, "!Fastboot: boot archive is too big"); goto err_out; } else { /* Set the flag so we don't keep retrying */ is_retry++; /* Adjust dma_attr_addr_lo */ fastboot_dma_attr.dma_attr_addr_lo = end_addr; fastboot_below_1G_dma_attr.dma_attr_addr_lo = end_addr; /* * Free the memory we have already allocated * whose physical addresses might not fit * the new lo and hi constraints. */ fastboot_free_mem(&newkernel, end_addr); goto load_kernel_retry; } } if (!fastboot_contig) dma_attr.dma_attr_sgllen = (fsize / PAGESIZE) + (((fsize % PAGESIZE) == 0) ? 0 : 1); if ((buf = contig_alloc(fsize, &dma_attr, PAGESIZE, 0)) == NULL) { cmn_err(CE_NOTE, fastboot_enomem_msg, fsize, "64G"); goto err_out; } va = P2ROUNDUP_TYPED((uintptr_t)buf, PAGESIZE, uintptr_t); if (kobj_read_file(file, (char *)va, fsize, 0) < 0) { cmn_err(CE_NOTE, "!Fastboot: Couldn't read %s", fastboot_filename[i]); goto err_out; } fb = &newkernel.fi_files[i]; fb->fb_va = va; fb->fb_size = fsize; fb->fb_sectcnt = 0; pt_size = FASTBOOT_PTE_LIST_SIZE(fsize_roundup); /* * If we have reserved memory but it not enough, free it. */ if (fb->fb_pte_list_size && fb->fb_pte_list_size < pt_size) { contig_free((void *)fb->fb_pte_list_va, fb->fb_pte_list_size); fb->fb_pte_list_size = 0; } if (fb->fb_pte_list_size == 0) { if ((fb->fb_pte_list_va = (x86pte_t *)contig_alloc(pt_size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) { cmn_err(CE_NOTE, fastboot_enomem_msg, (uint64_t)pt_size, "1G"); goto err_out; } /* * fb_pte_list_size must be set after the allocation * succeeds as it's used to determine how much memory to * free. */ fb->fb_pte_list_size = pt_size; } bzero((void *)(fb->fb_pte_list_va), fb->fb_pte_list_size); fb->fb_pte_list_pa = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat, (caddr_t)fb->fb_pte_list_va)); for (page_index = 0, offset = 0; offset < fb->fb_size; offset += PAGESIZE) { uint64_t paddr; paddr = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat, (caddr_t)fb->fb_va + offset)); ASSERT(paddr >= fastboot_dma_attr.dma_attr_addr_lo); /* * Include the pte_bits so we don't have to make * it in assembly. */ fb->fb_pte_list_va[page_index++] = (x86pte_t) (paddr | pte_bits); } fb->fb_pte_list_va[page_index] = FASTBOOT_TERMINATE; if (i == FASTBOOT_UNIX) { Ehdr *ehdr = (Ehdr *)va; int j; /* * Sanity checks: */ for (j = 0; j < SELFMAG; j++) { if (ehdr->e_ident[j] != ELFMAG[j]) { cmn_err(CE_NOTE, "!Fastboot: Bad ELF " "signature"); goto err_out; } } if (ehdr->e_ident[EI_CLASS] == ELFCLASS32 && ehdr->e_ident[EI_DATA] == ELFDATA2LSB && ehdr->e_machine == EM_386) { fb->fb_sectcnt = sizeof (fb->fb_sections) / sizeof (fb->fb_sections[0]); if (fastboot_elf32_find_loadables((void *)va, fsize, &fb->fb_sections[0], &fb->fb_sectcnt, &dboot_start_offset) < 0) { cmn_err(CE_NOTE, "!Fastboot: ELF32 " "program section failure"); goto err_out; } if (fb->fb_sectcnt == 0) { cmn_err(CE_NOTE, "!Fastboot: No ELF32 " "program sections found"); goto err_out; } if (is_failsafe) { /* Failsafe boot_archive */ bcopy(BOOTARCHIVE32_FAILSAFE, &fastboot_filename [FASTBOOT_NAME_BOOTARCHIVE] [bootpath_len], sizeof (BOOTARCHIVE32_FAILSAFE)); } else { bcopy(BOOTARCHIVE32, &fastboot_filename [FASTBOOT_NAME_BOOTARCHIVE] [bootpath_len], sizeof (BOOTARCHIVE32)); } } else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64 && ehdr->e_ident[EI_DATA] == ELFDATA2LSB && ehdr->e_machine == EM_AMD64) { if (fastboot_elf64_find_dboot_load_offset( (void *)va, fsize, &dboot_start_offset) != 0) { cmn_err(CE_NOTE, "!Fastboot: Couldn't " "find ELF64 dboot entry offset"); goto err_out; } if (!is_x86_feature(x86_featureset, X86FSET_64) || !is_x86_feature(x86_featureset, X86FSET_PAE)) { cmn_err(CE_NOTE, "Fastboot: Cannot " "reboot to %s: " "not a 64-bit capable system", kern_bootfile); goto err_out; } if (is_failsafe) { /* Failsafe boot_archive */ bcopy(BOOTARCHIVE64_FAILSAFE, &fastboot_filename [FASTBOOT_NAME_BOOTARCHIVE] [bootpath_len], sizeof (BOOTARCHIVE64_FAILSAFE)); } else { bcopy(BOOTARCHIVE64, &fastboot_filename [FASTBOOT_NAME_BOOTARCHIVE] [bootpath_len], sizeof (BOOTARCHIVE64)); } } else { cmn_err(CE_NOTE, "!Fastboot: Unknown ELF type"); goto err_out; } fb->fb_dest_pa = DBOOT_ENTRY_ADDRESS - dboot_start_offset; fb->fb_next_pa = DBOOT_ENTRY_ADDRESS + fsize_roundup; } else { fb->fb_dest_pa = newkernel.fi_files[i - 1].fb_next_pa; fb->fb_next_pa = fb->fb_dest_pa + fsize_roundup; } kobj_close_file(file); } /* * Add the function that will switch us to 32-bit protected mode */ fb = &newkernel.fi_files[FASTBOOT_SWTCH]; fb->fb_va = fb->fb_dest_pa = FASTBOOT_SWTCH_PA; fb->fb_size = MMU_PAGESIZE; hat_devload(kas.a_hat, (caddr_t)fb->fb_va, MMU_PAGESIZE, mmu_btop(fb->fb_dest_pa), PROT_READ | PROT_WRITE | PROT_EXEC, HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK); /* * Build the new multiboot_info structure */ if (fastboot_build_mbi(fastboot_args, &newkernel) != 0) { goto err_out; } /* * Build page table for low 1G physical memory. Use big pages. * Allocate 4 (5 for amd64) pages for the page tables. * 1 page for PML4 (amd64) * 1 page for Page-Directory-Pointer Table * 2 pages for Page Directory * 1 page for Page Table. * The page table entry will be rewritten to map the physical * address as we do the copying. */ if (newkernel.fi_has_pae) { #ifdef __amd64 size_t size = MMU_PAGESIZE * 5; #else size_t size = MMU_PAGESIZE * 4; #endif /* __amd64 */ if (newkernel.fi_pagetable_size && newkernel.fi_pagetable_size < size) { contig_free((void *)newkernel.fi_pagetable_va, newkernel.fi_pagetable_size); newkernel.fi_pagetable_size = 0; } if (newkernel.fi_pagetable_size == 0) { if ((newkernel.fi_pagetable_va = (uintptr_t) contig_alloc(size, &fastboot_below_1G_dma_attr, MMU_PAGESIZE, 0)) == NULL) { cmn_err(CE_NOTE, fastboot_enomem_msg, (uint64_t)size, "1G"); goto err_out; } /* * fi_pagetable_size must be set after the allocation * succeeds as it's used to determine how much memory to * free. */ newkernel.fi_pagetable_size = size; } bzero((void *)(newkernel.fi_pagetable_va), size); newkernel.fi_pagetable_pa = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat, (caddr_t)newkernel.fi_pagetable_va)); newkernel.fi_last_table_pa = newkernel.fi_pagetable_pa + size - MMU_PAGESIZE; newkernel.fi_next_table_va = newkernel.fi_pagetable_va + MMU_PAGESIZE; newkernel.fi_next_table_pa = newkernel.fi_pagetable_pa + MMU_PAGESIZE; fastboot_build_pagetables(&newkernel); } /* Generate MD5 checksums */ fastboot_cksum_generate(&newkernel); /* Mark it as valid */ newkernel.fi_valid = 1; newkernel.fi_magic = FASTBOOT_MAGIC; postbootkernelbase = saved_kernelbase; return; err_out: postbootkernelbase = saved_kernelbase; newkernel.fi_valid = 0; fastboot_free_newkernel(&newkernel); }
/* * Create multiboot info structure (mbi) base on the saved mbi. * Recalculate values of the pointer type fields in the data * structure based on the new starting physical address of the * data structure. */ static int fastboot_build_mbi(char *mdep, fastboot_info_t *nk) { mb_module_t *mbp; multiboot_info_t *mbi; /* pointer to multiboot structure */ uintptr_t start_addr_va; /* starting VA of mbi */ uintptr_t start_addr_pa; /* starting PA of mbi */ size_t offs = 0; /* offset from the starting address */ size_t arglen; /* length of the command line arg */ size_t size; /* size of the memory reserved for mbi */ size_t mdnsz; /* length of the boot archive name */ /* * If mdep is not NULL or empty, use the length of mdep + 1 * (for NULL terminating) as the length of the new command * line; else use the saved command line length as the * length for the new command line. */ if (mdep != NULL && strlen(mdep) != 0) { arglen = strlen(mdep) + 1; } else { arglen = saved_cmdline_len; } /* * Allocate memory for the new multiboot info structure (mbi). * If we have reserved memory for mbi but it's not enough, * free it and reallocate. */ size = PAGESIZE + P2ROUNDUP(arglen, PAGESIZE); if (nk->fi_mbi_size && nk->fi_mbi_size < size) { contig_free((void *)nk->fi_new_mbi_va, nk->fi_mbi_size); nk->fi_mbi_size = 0; } if (nk->fi_mbi_size == 0) { if ((nk->fi_new_mbi_va = (uintptr_t)contig_alloc(size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) { cmn_err(CE_NOTE, fastboot_enomem_msg, (uint64_t)size, "1G"); return (-1); } /* * fi_mbi_size must be set after the allocation succeeds * as it's used to determine how much memory to free. */ nk->fi_mbi_size = size; } /* * Initalize memory */ bzero((void *)nk->fi_new_mbi_va, nk->fi_mbi_size); /* * Get PA for the new mbi */ start_addr_va = nk->fi_new_mbi_va; start_addr_pa = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat, (caddr_t)start_addr_va)); nk->fi_new_mbi_pa = (paddr_t)start_addr_pa; /* * Populate the rest of the fields in the data structure */ /* * Copy from the saved mbi to preserve all non-pointer type fields. */ mbi = (multiboot_info_t *)start_addr_va; bcopy(&saved_mbi, mbi, sizeof (*mbi)); /* * Recalculate mods_addr. Set mod_start and mod_end based on * the physical address of the new boot archive. Set mod_name * to the name of the new boto archive. */ offs += sizeof (multiboot_info_t); mbi->mods_addr = start_addr_pa + offs; mbp = (mb_module_t *)(start_addr_va + offs); mbp->mod_start = nk->fi_files[FASTBOOT_BOOTARCHIVE].fb_dest_pa; mbp->mod_end = nk->fi_files[FASTBOOT_BOOTARCHIVE].fb_next_pa; offs += sizeof (mb_module_t); mdnsz = strlen(fastboot_filename[FASTBOOT_NAME_BOOTARCHIVE]) + 1; bcopy(fastboot_filename[FASTBOOT_NAME_BOOTARCHIVE], (void *)(start_addr_va + offs), mdnsz); mbp->mod_name = start_addr_pa + offs; mbp->reserved = 0; /* * Make sure the offset is 16-byte aligned to avoid unaligned access. */ offs += mdnsz; offs = P2ROUNDUP_TYPED(offs, 16, size_t); /* * Recalculate mmap_addr */ mbi->mmap_addr = start_addr_pa + offs; bcopy((void *)(uintptr_t)saved_mmap, (void *)(start_addr_va + offs), saved_mbi.mmap_length); offs += saved_mbi.mmap_length; /* * Recalculate drives_addr */ mbi->drives_addr = start_addr_pa + offs; bcopy((void *)(uintptr_t)saved_drives, (void *)(start_addr_va + offs), saved_mbi.drives_length); offs += saved_mbi.drives_length; /* * Recalculate the address of cmdline. Set cmdline to contain the * new boot argument. */ mbi->cmdline = start_addr_pa + offs; if (mdep != NULL && strlen(mdep) != 0) { bcopy(mdep, (void *)(start_addr_va + offs), arglen); } else { bcopy((void *)saved_cmdline, (void *)(start_addr_va + offs), arglen); } /* clear fields and flags that are not copied */ bzero(&mbi->config_table, sizeof (*mbi) - offsetof(multiboot_info_t, config_table)); mbi->flags &= ~(MB_INFO_CONFIG_TABLE | MB_INFO_BOOT_LOADER_NAME | MB_INFO_APM_TABLE | MB_INFO_VIDEO_INFO); return (0); }