/* * kobj_jettison: * * Release object data not needed after performing relocations. */ static void kobj_jettison(kobj_t ko) { int i; if (ko->ko_reltab != NULL) { for (i = 0; i < ko->ko_nrel; i++) { if (ko->ko_reltab[i].rel) { kobj_free(ko, ko->ko_reltab[i].rel, ko->ko_reltab[i].size); } } kobj_free(ko, ko->ko_reltab, ko->ko_nrel * sizeof(*ko->ko_reltab)); ko->ko_reltab = NULL; ko->ko_nrel = 0; } if (ko->ko_relatab != NULL) { for (i = 0; i < ko->ko_nrela; i++) { if (ko->ko_relatab[i].rela) { kobj_free(ko, ko->ko_relatab[i].rela, ko->ko_relatab[i].size); } } kobj_free(ko, ko->ko_relatab, ko->ko_nrela * sizeof(*ko->ko_relatab)); ko->ko_relatab = NULL; ko->ko_nrela = 0; } if (ko->ko_shdr != NULL) { kobj_free(ko, ko->ko_shdr, ko->ko_shdrsz); ko->ko_shdr = NULL; } }
/*ARGSUSED*/ void bkmem_free(void *p, size_t size) { /* * Free only if it's not boot scratch memory. */ if ((uintptr_t)p >= scratch_max) kobj_free(p, size); }
/*ARGSUSED*/ void zcfree(void *opaque, void *ptr) { struct zchdr *z = ((struct zchdr *)ptr) - 1; if (z->zch_magic != ZCH_MAGIC) panic("zcfree region corrupt: hdr=%p ptr=%p", (void *)z, ptr); kobj_free(z, z->zch_size); }
/* * kobj_unload: * * Unload an object previously loaded by kobj_load(). */ void kobj_unload(kobj_t ko) { int error; kobj_close(ko); kobj_jettison(ko); /* * Notify MD code that a module has been unloaded. */ if (ko->ko_loaded) { error = kobj_machdep(ko, (void *)ko->ko_address, ko->ko_size, false); if (error != 0) kobj_error(__func__, __LINE__, ko, "machine dependent deinit failed %d", error); } if (ko->ko_address != 0 && ko->ko_type != KT_MEMORY) { uvm_km_free(module_map, ko->ko_address, round_page(ko->ko_size), UVM_KMF_WIRED); } if (ko->ko_ksyms == true) { ksyms_modunload(ko->ko_name); } if (ko->ko_symtab != NULL) { kobj_free(ko, ko->ko_symtab, ko->ko_symcnt * sizeof(Elf_Sym)); } if (ko->ko_strtab != NULL) { kobj_free(ko, ko->ko_strtab, ko->ko_strtabsz); } if (ko->ko_progtab != NULL) { kobj_free(ko, ko->ko_progtab, ko->ko_nprogtab * sizeof(*ko->ko_progtab)); ko->ko_progtab = NULL; } if (ko->ko_shstrtab) { kobj_free(ko, ko->ko_shstrtab, ko->ko_shstrtabsz); ko->ko_shstrtab = NULL; } kmem_free(ko, sizeof(*ko)); }
/*ARGSUSED*/ void bkmem_free(void *p, size_t size) { /* * Don't bother freeing scratch memory * Note that in amd64, BOP_ALLOC returns address * prepended with 0xffffffff, so we cast to 32-bit * before comparing. */ if ((uint32_t)(uintptr_t)p > MAGIC_PHYS) kobj_free(p, size); }
/*ARGSUSED*/ void ctf_free(void *buf, size_t size) { kobj_free(buf, size); }
int do_relocations(struct module *mp) { uint_t shn; Shdr *shp, *rshp; uint_t nreloc; /* do the relocations */ for (shn = 1; shn < mp->hdr.e_shnum; shn++) { rshp = (Shdr *) (mp->shdrs + shn * mp->hdr.e_shentsize); if (rshp->sh_type == SHT_RELA) { _kobj_printf(ops, "%s can't process type SHT_RELA\n", mp->filename); return (-1); } if (rshp->sh_type != SHT_REL) continue; if (rshp->sh_link != mp->symtbl_section) { _kobj_printf(ops, "%s reloc for non-default symtab\n", mp->filename); return (-1); } if (rshp->sh_info >= mp->hdr.e_shnum) { _kobj_printf(ops, "do_relocations: %s sh_info ", mp->filename); _kobj_printf(ops, "out of range %d\n", shn); goto bad; } nreloc = rshp->sh_size / rshp->sh_entsize; /* get the section header that this reloc table refers to */ shp = (Shdr *) (mp->shdrs + rshp->sh_info * mp->hdr.e_shentsize); /* * Do not relocate any section that isn't loaded into memory. * Most commonly this will skip over the .rela.stab* sections */ if (!(shp->sh_flags & SHF_ALLOC)) continue; #ifdef KOBJ_DEBUG if (kobj_debug & D_RELOCATIONS) { _kobj_printf(ops, "krtld: relocating: file=%s ", mp->filename); _kobj_printf(ops, "section=%d\n", shn); } #endif if (do_relocate(mp, (char *)rshp->sh_addr, rshp->sh_type, nreloc, rshp->sh_entsize, shp->sh_addr) < 0) { _kobj_printf(ops, "do_relocations: %s do_relocate failed\n", mp->filename); goto bad; } kobj_free((void *)rshp->sh_addr, rshp->sh_size); rshp->sh_addr = 0; } mp->flags |= KOBJ_RELOCATED; return (0); bad: kobj_free((void *)rshp->sh_addr, rshp->sh_size); rshp->sh_addr = 0; return (-1); }
void kctl_strfree(char *s) { kobj_free(s, strlen(s) + 1); }
/* * kobj_load: * * Load an ELF object and prepare to link into the running kernel * image. */ int kobj_load(kobj_t ko) { Elf_Ehdr *hdr; Elf_Shdr *shdr; Elf_Sym *es; vaddr_t mapbase; size_t mapsize; int error; int symtabindex; int symstrindex; int nsym; int pb, rl, ra; int alignmask; int i, j; void *addr; KASSERT(ko->ko_type != KT_UNSET); KASSERT(ko->ko_source != NULL); shdr = NULL; mapsize = 0; error = 0; hdr = NULL; /* * Read the elf header from the file. */ error = ko->ko_read(ko, (void **)&hdr, sizeof(*hdr), 0, true); if (error != 0) { kobj_error(__func__, __LINE__, ko, "read failed %d", error); goto out; } if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0) { kobj_error(__func__, __LINE__, ko, "not an ELF object"); error = ENOEXEC; goto out; } if (hdr->e_ident[EI_VERSION] != EV_CURRENT || hdr->e_version != EV_CURRENT) { kobj_error(__func__, __LINE__, ko, "unsupported file version %d", hdr->e_ident[EI_VERSION]); error = ENOEXEC; goto out; } if (hdr->e_type != ET_REL) { kobj_error(__func__, __LINE__, ko, "unsupported file type %d", hdr->e_type); error = ENOEXEC; goto out; } switch (hdr->e_machine) { #if ELFSIZE == 32 ELF32_MACHDEP_ID_CASES #elif ELFSIZE == 64 ELF64_MACHDEP_ID_CASES #else #error not defined #endif default: kobj_error(__func__, __LINE__, ko, "unsupported machine %d", hdr->e_machine); error = ENOEXEC; goto out; } ko->ko_nprogtab = 0; ko->ko_shdr = 0; ko->ko_nrel = 0; ko->ko_nrela = 0; /* * Allocate and read in the section header. */ ko->ko_shdrsz = hdr->e_shnum * hdr->e_shentsize; if (ko->ko_shdrsz == 0 || hdr->e_shoff == 0 || hdr->e_shentsize != sizeof(Elf_Shdr)) { kobj_error(__func__, __LINE__, ko, "bad sizes"); error = ENOEXEC; goto out; } error = ko->ko_read(ko, (void **)&shdr, ko->ko_shdrsz, hdr->e_shoff, true); if (error != 0) { kobj_error(__func__, __LINE__, ko, "read failed %d", error); goto out; } ko->ko_shdr = shdr; /* * Scan the section header for information and table sizing. */ nsym = 0; symtabindex = -1; symstrindex = -1; for (i = 0; i < hdr->e_shnum; i++) { switch (shdr[i].sh_type) { case SHT_PROGBITS: case SHT_NOBITS: ko->ko_nprogtab++; break; case SHT_SYMTAB: nsym++; symtabindex = i; symstrindex = shdr[i].sh_link; break; case SHT_REL: if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) continue; ko->ko_nrel++; break; case SHT_RELA: if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) continue; ko->ko_nrela++; break; case SHT_STRTAB: break; } } if (ko->ko_nprogtab == 0) { kobj_error(__func__, __LINE__, ko, "file has no contents"); error = ENOEXEC; goto out; } if (nsym != 1) { /* Only allow one symbol table for now */ kobj_error(__func__, __LINE__, ko, "file has no valid symbol table"); error = ENOEXEC; goto out; } if (symstrindex < 0 || symstrindex > hdr->e_shnum || shdr[symstrindex].sh_type != SHT_STRTAB) { kobj_error(__func__, __LINE__, ko, "file has invalid symbol strings"); error = ENOEXEC; goto out; } /* * Allocate space for tracking the load chunks. */ if (ko->ko_nprogtab != 0) { ko->ko_progtab = kmem_zalloc(ko->ko_nprogtab * sizeof(*ko->ko_progtab), KM_SLEEP); if (ko->ko_progtab == NULL) { error = ENOMEM; kobj_error(__func__, __LINE__, ko, "out of memory"); goto out; } } if (ko->ko_nrel != 0) { ko->ko_reltab = kmem_zalloc(ko->ko_nrel * sizeof(*ko->ko_reltab), KM_SLEEP); if (ko->ko_reltab == NULL) { error = ENOMEM; kobj_error(__func__, __LINE__, ko, "out of memory"); goto out; } } if (ko->ko_nrela != 0) { ko->ko_relatab = kmem_zalloc(ko->ko_nrela * sizeof(*ko->ko_relatab), KM_SLEEP); if (ko->ko_relatab == NULL) { error = ENOMEM; kobj_error(__func__, __LINE__, ko, "out of memory"); goto out; } } if (symtabindex == -1) { kobj_error(__func__, __LINE__, ko, "lost symbol table index"); goto out; } /* * Allocate space for and load the symbol table. */ ko->ko_symcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym); if (ko->ko_symcnt == 0) { kobj_error(__func__, __LINE__, ko, "no symbol table"); goto out; } error = ko->ko_read(ko, (void **)&ko->ko_symtab, ko->ko_symcnt * sizeof(Elf_Sym), shdr[symtabindex].sh_offset, true); if (error != 0) { kobj_error(__func__, __LINE__, ko, "read failed %d", error); goto out; } /* * Allocate space for and load the symbol strings. */ ko->ko_strtabsz = shdr[symstrindex].sh_size; if (ko->ko_strtabsz == 0) { kobj_error(__func__, __LINE__, ko, "no symbol strings"); goto out; } error = ko->ko_read(ko, (void *)&ko->ko_strtab, ko->ko_strtabsz, shdr[symstrindex].sh_offset, true); if (error != 0) { kobj_error(__func__, __LINE__, ko, "read failed %d", error); goto out; } /* * Adjust module symbol namespace, if necessary (e.g. with rump) */ error = kobj_renamespace(ko->ko_symtab, ko->ko_symcnt, &ko->ko_strtab, &ko->ko_strtabsz); if (error != 0) { kobj_error(__func__, __LINE__, ko, "renamespace failed %d", error); goto out; } /* * Do we have a string table for the section names? */ if (hdr->e_shstrndx != 0 && shdr[hdr->e_shstrndx].sh_size != 0 && shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) { ko->ko_shstrtabsz = shdr[hdr->e_shstrndx].sh_size; error = ko->ko_read(ko, (void **)&ko->ko_shstrtab, shdr[hdr->e_shstrndx].sh_size, shdr[hdr->e_shstrndx].sh_offset, true); if (error != 0) { kobj_error(__func__, __LINE__, ko, "read failed %d", error); goto out; } } /* * Size up code/data(progbits) and bss(nobits). */ alignmask = 0; mapbase = 0; for (i = 0; i < hdr->e_shnum; i++) { switch (shdr[i].sh_type) { case SHT_PROGBITS: case SHT_NOBITS: if (mapbase == 0) mapbase = shdr[i].sh_offset; alignmask = shdr[i].sh_addralign - 1; mapsize += alignmask; mapsize &= ~alignmask; mapsize += shdr[i].sh_size; break; } } /* * We know how much space we need for the text/data/bss/etc. * This stuff needs to be in a single chunk so that profiling etc * can get the bounds and gdb can associate offsets with modules. */ if (mapsize == 0) { kobj_error(__func__, __LINE__, ko, "no text/data/bss"); goto out; } if (ko->ko_type == KT_MEMORY) { mapbase += (vaddr_t)ko->ko_source; } else { mapbase = uvm_km_alloc(module_map, round_page(mapsize), 0, UVM_KMF_WIRED | UVM_KMF_EXEC); if (mapbase == 0) { kobj_error(__func__, __LINE__, ko, "out of memory"); error = ENOMEM; goto out; } } ko->ko_address = mapbase; ko->ko_size = mapsize; /* * Now load code/data(progbits), zero bss(nobits), allocate space * for and load relocs */ pb = 0; rl = 0; ra = 0; alignmask = 0; for (i = 0; i < hdr->e_shnum; i++) { switch (shdr[i].sh_type) { case SHT_PROGBITS: case SHT_NOBITS: alignmask = shdr[i].sh_addralign - 1; if (ko->ko_type == KT_MEMORY) { addr = (void *)(shdr[i].sh_offset + (vaddr_t)ko->ko_source); if (((vaddr_t)addr & alignmask) != 0) { kobj_error(__func__, __LINE__, ko, "section %d not aligned", i); goto out; } } else { mapbase += alignmask; mapbase &= ~alignmask; addr = (void *)mapbase; mapbase += shdr[i].sh_size; } ko->ko_progtab[pb].addr = addr; if (shdr[i].sh_type == SHT_PROGBITS) { ko->ko_progtab[pb].name = "<<PROGBITS>>"; error = ko->ko_read(ko, &addr, shdr[i].sh_size, shdr[i].sh_offset, false); if (error != 0) { kobj_error(__func__, __LINE__, ko, "read failed %d", error); goto out; } } else if (ko->ko_type == KT_MEMORY && shdr[i].sh_size != 0) { kobj_error(__func__, __LINE__, ko, "non-loadable BSS " "section in pre-loaded module"); error = EINVAL; goto out; } else { ko->ko_progtab[pb].name = "<<NOBITS>>"; memset(addr, 0, shdr[i].sh_size); } ko->ko_progtab[pb].size = shdr[i].sh_size; ko->ko_progtab[pb].sec = i; if (ko->ko_shstrtab != NULL && shdr[i].sh_name != 0) { ko->ko_progtab[pb].name = ko->ko_shstrtab + shdr[i].sh_name; } /* Update all symbol values with the offset. */ for (j = 0; j < ko->ko_symcnt; j++) { es = &ko->ko_symtab[j]; if (es->st_shndx != i) { continue; } es->st_value += (Elf_Addr)addr; } pb++; break; case SHT_REL: if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) break; ko->ko_reltab[rl].size = shdr[i].sh_size; ko->ko_reltab[rl].size -= shdr[i].sh_size % sizeof(Elf_Rel); if (ko->ko_reltab[rl].size != 0) { ko->ko_reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel); ko->ko_reltab[rl].sec = shdr[i].sh_info; error = ko->ko_read(ko, (void **)&ko->ko_reltab[rl].rel, ko->ko_reltab[rl].size, shdr[i].sh_offset, true); if (error != 0) { kobj_error(__func__, __LINE__, ko, "read failed %d", error); goto out; } } rl++; break; case SHT_RELA: if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) break; ko->ko_relatab[ra].size = shdr[i].sh_size; ko->ko_relatab[ra].size -= shdr[i].sh_size % sizeof(Elf_Rela); if (ko->ko_relatab[ra].size != 0) { ko->ko_relatab[ra].nrela = shdr[i].sh_size / sizeof(Elf_Rela); ko->ko_relatab[ra].sec = shdr[i].sh_info; error = ko->ko_read(ko, (void **)&ko->ko_relatab[ra].rela, shdr[i].sh_size, shdr[i].sh_offset, true); if (error != 0) { kobj_error(__func__, __LINE__, ko, "read failed %d", error); goto out; } } ra++; break; default: break; } } if (pb != ko->ko_nprogtab) { panic("%s:%d: %s: lost progbits", __func__, __LINE__, ko->ko_name); } if (rl != ko->ko_nrel) { panic("%s:%d: %s: lost rel", __func__, __LINE__, ko->ko_name); } if (ra != ko->ko_nrela) { panic("%s:%d: %s: lost rela", __func__, __LINE__, ko->ko_name); } if (ko->ko_type != KT_MEMORY && mapbase != ko->ko_address + mapsize) { panic("%s:%d: %s: " "mapbase 0x%lx != address %lx + mapsize %ld (0x%lx)\n", __func__, __LINE__, ko->ko_name, (long)mapbase, (long)ko->ko_address, (long)mapsize, (long)ko->ko_address + mapsize); } /* * Perform local relocations only. Relocations relating to global * symbols will be done by kobj_affix(). */ error = kobj_checksyms(ko, false); if (error == 0) { error = kobj_relocate(ko, true); } out: if (hdr != NULL) { kobj_free(ko, hdr, sizeof(*hdr)); } kobj_close(ko); if (error != 0) { kobj_unload(ko); } return error; }