static int
link_elf_preload_file(const char *filename, linker_file_t *result)
{
caddr_t modptr, baseptr, sizeptr, dynptr;
char *type;
elf_file_t ef;
linker_file_t lf;
int error;
vm_offset_t dp;
/*
* Look to see if we have the module preloaded.
*/
modptr = preload_search_by_name(filename);
if (modptr == NULL)
return ENOENT;
/* It's preloaded, check we can handle it and collect information */
type = (char *)preload_search_info(modptr, MODINFO_TYPE);
baseptr = preload_search_info(modptr, MODINFO_ADDR);
sizeptr = preload_search_info(modptr, MODINFO_SIZE);
dynptr = preload_search_info(modptr, MODINFO_METADATA|MODINFOMD_DYNAMIC);
if (type == NULL ||
(strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) " module") != 0 &&
strcmp(type, "elf module") != 0))
return (EFTYPE);
if (baseptr == NULL || sizeptr == NULL || dynptr == NULL)
return (EINVAL);
ef = kmalloc(sizeof(struct elf_file), M_LINKER, M_WAITOK | M_ZERO);
ef->modptr = modptr;
ef->address = *(caddr_t *)baseptr;
#ifdef SPARSE_MAPPING
ef->object = NULL;
#endif
dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr;
ef->dynamic = (Elf_Dyn *)dp;
lf = linker_make_file(filename, ef, &link_elf_module_ops);
if (lf == NULL) {
kfree(ef, M_LINKER);
return ENOMEM;
}
lf->address = ef->address;
lf->size = *(size_t *)sizeptr;
error = parse_dynamic(lf);
if (error) {
linker_file_unload(lf);
return error;
}
link_elf_reloc_local(lf);
*result = lf;
return (0);
}
static void
link_elf_init(void* arg)
{
Elf_Dyn *dp;
caddr_t modptr, baseptr, sizeptr;
elf_file_t ef;
char *modname;
#if ELF_TARG_CLASS == ELFCLASS32
linker_add_class("elf32", NULL, &link_elf_class_ops);
#else
linker_add_class("elf64", NULL, &link_elf_class_ops);
#endif
dp = (Elf_Dyn*) &_DYNAMIC;
if (dp) {
ef = kmalloc(sizeof(struct elf_file), M_LINKER, M_INTWAIT | M_ZERO);
ef->address = NULL;
#ifdef SPARSE_MAPPING
ef->object = NULL;
#endif
ef->dynamic = dp;
modname = NULL;
modptr = preload_search_by_type("elf kernel");
if (modptr)
modname = (char *)preload_search_info(modptr, MODINFO_NAME);
if (modname == NULL)
modname = "kernel";
linker_kernel_file = linker_make_file(modname, ef, &link_elf_file_ops);
if (linker_kernel_file == NULL)
panic("link_elf_init: Can't create linker structures for kernel");
parse_dynamic(linker_kernel_file);
#if defined(__amd64__) && defined(_KERNEL_VIRTUAL)
fprintf(stderr, "WARNING: KERNBASE being used\n");
#endif
linker_kernel_file->address = (caddr_t) KERNBASE;
linker_kernel_file->size = -(intptr_t)linker_kernel_file->address;
if (modptr) {
ef->modptr = modptr;
baseptr = preload_search_info(modptr, MODINFO_ADDR);
if (baseptr)
linker_kernel_file->address = *(caddr_t *)baseptr;
sizeptr = preload_search_info(modptr, MODINFO_SIZE);
if (sizeptr)
linker_kernel_file->size = *(size_t *)sizeptr;
}
parse_module_symbols(linker_kernel_file);
linker_current_file = linker_kernel_file;
linker_kernel_file->flags |= LINKER_FILE_LINKED;
}
}
static int
link_elf_load_file(linker_class_t cls, const char *filename,
linker_file_t *result)
{
struct nameidata nd;
struct thread *td = curthread; /* XXX */
Elf_Ehdr *hdr;
Elf_Shdr *shdr;
Elf_Sym *es;
int nbytes, i, j;
vm_offset_t mapbase;
size_t mapsize;
int error = 0;
ssize_t resid;
int flags;
elf_file_t ef;
linker_file_t lf;
int symtabindex;
int symstrindex;
int shstrindex;
int nsym;
int pb, rl, ra;
int alignmask;
shdr = NULL;
lf = NULL;
mapsize = 0;
hdr = NULL;
NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td);
flags = FREAD;
error = vn_open(&nd, &flags, 0, NULL);
if (error)
return error;
NDFREE(&nd, NDF_ONLY_PNBUF);
if (nd.ni_vp->v_type != VREG) {
error = ENOEXEC;
goto out;
}
#ifdef MAC
error = mac_kld_check_load(td->td_ucred, nd.ni_vp);
if (error) {
goto out;
}
#endif
/* Read the elf header from the file. */
hdr = malloc(sizeof(*hdr), M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)hdr, sizeof(*hdr), 0,
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
&resid, td);
if (error)
goto out;
if (resid != 0){
error = ENOEXEC;
goto out;
}
if (!IS_ELF(*hdr)) {
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS
|| hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
link_elf_error(filename, "Unsupported file layout");
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_VERSION] != EV_CURRENT
|| hdr->e_version != EV_CURRENT) {
link_elf_error(filename, "Unsupported file version");
error = ENOEXEC;
goto out;
}
if (hdr->e_type != ET_REL) {
error = ENOSYS;
goto out;
}
if (hdr->e_machine != ELF_TARG_MACH) {
link_elf_error(filename, "Unsupported machine");
error = ENOEXEC;
goto out;
}
lf = linker_make_file(filename, &link_elf_class);
if (!lf) {
error = ENOMEM;
goto out;
}
ef = (elf_file_t) lf;
ef->nprogtab = 0;
ef->e_shdr = 0;
ef->nreltab = 0;
ef->nrelatab = 0;
/* Allocate and read in the section header */
nbytes = hdr->e_shnum * hdr->e_shentsize;
if (nbytes == 0 || hdr->e_shoff == 0 ||
hdr->e_shentsize != sizeof(Elf_Shdr)) {
error = ENOEXEC;
goto out;
}
shdr = malloc(nbytes, M_LINKER, M_WAITOK);
ef->e_shdr = shdr;
error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shdr, nbytes, hdr->e_shoff,
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td);
if (error)
goto out;
if (resid) {
error = ENOEXEC;
goto out;
}
/* Scan the section header for information and table sizing. */
nsym = 0;
symtabindex = -1;
symstrindex = -1;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_size == 0)
continue;
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
ef->nprogtab++;
break;
case SHT_SYMTAB:
nsym++;
symtabindex = i;
symstrindex = shdr[i].sh_link;
break;
case SHT_REL:
ef->nreltab++;
break;
case SHT_RELA:
ef->nrelatab++;
break;
case SHT_STRTAB:
break;
}
}
if (ef->nprogtab == 0) {
link_elf_error(filename, "file has no contents");
error = ENOEXEC;
goto out;
}
if (nsym != 1) {
/* Only allow one symbol table for now */
link_elf_error(filename, "file has no valid symbol table");
error = ENOEXEC;
goto out;
}
if (symstrindex < 0 || symstrindex > hdr->e_shnum ||
shdr[symstrindex].sh_type != SHT_STRTAB) {
link_elf_error(filename, "file has invalid symbol strings");
error = ENOEXEC;
goto out;
}
/* Allocate space for tracking the load chunks */
if (ef->nprogtab != 0)
ef->progtab = malloc(ef->nprogtab * sizeof(*ef->progtab),
M_LINKER, M_WAITOK | M_ZERO);
if (ef->nreltab != 0)
ef->reltab = malloc(ef->nreltab * sizeof(*ef->reltab),
M_LINKER, M_WAITOK | M_ZERO);
if (ef->nrelatab != 0)
ef->relatab = malloc(ef->nrelatab * sizeof(*ef->relatab),
M_LINKER, M_WAITOK | M_ZERO);
if (symtabindex == -1)
panic("lost symbol table index");
/* Allocate space for and load the symbol table */
ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym);
ef->ddbsymtab = malloc(shdr[symtabindex].sh_size, M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, nd.ni_vp, (void *)ef->ddbsymtab,
shdr[symtabindex].sh_size, shdr[symtabindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
&resid, td);
if (error)
goto out;
if (resid != 0){
error = EINVAL;
goto out;
}
if (symstrindex == -1)
panic("lost symbol string index");
/* Allocate space for and load the symbol strings */
ef->ddbstrcnt = shdr[symstrindex].sh_size;
ef->ddbstrtab = malloc(shdr[symstrindex].sh_size, M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, nd.ni_vp, ef->ddbstrtab,
shdr[symstrindex].sh_size, shdr[symstrindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
&resid, td);
if (error)
goto out;
if (resid != 0){
error = EINVAL;
goto out;
}
/* Do we have a string table for the section names? */
shstrindex = -1;
if (hdr->e_shstrndx != 0 &&
shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) {
shstrindex = hdr->e_shstrndx;
ef->shstrcnt = shdr[shstrindex].sh_size;
ef->shstrtab = malloc(shdr[shstrindex].sh_size, M_LINKER,
M_WAITOK);
error = vn_rdwr(UIO_READ, nd.ni_vp, ef->shstrtab,
shdr[shstrindex].sh_size, shdr[shstrindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
&resid, td);
if (error)
goto out;
if (resid != 0){
error = EINVAL;
goto out;
}
}
/* Size up code/data(progbits) and bss(nobits). */
alignmask = 0;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_size == 0)
continue;
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
alignmask = shdr[i].sh_addralign - 1;
mapsize += alignmask;
mapsize &= ~alignmask;
mapsize += shdr[i].sh_size;
break;
}
}
/*
* We know how much space we need for the text/data/bss/etc.
* This stuff needs to be in a single chunk so that profiling etc
* can get the bounds and gdb can associate offsets with modules
*/
ef->object = vm_object_allocate(OBJT_DEFAULT,
round_page(mapsize) >> PAGE_SHIFT);
if (ef->object == NULL) {
error = ENOMEM;
goto out;
}
ef->address = (caddr_t) vm_map_min(kernel_map);
/*
* In order to satisfy amd64's architectural requirements on the
* location of code and data in the kernel's address space, request a
* mapping that is above the kernel.
*/
#ifdef __amd64__
mapbase = KERNBASE;
#else
mapbase = VM_MIN_KERNEL_ADDRESS;
#endif
error = vm_map_find(kernel_map, ef->object, 0, &mapbase,
round_page(mapsize), 0, VMFS_OPTIMAL_SPACE, VM_PROT_ALL,
VM_PROT_ALL, 0);
if (error) {
vm_object_deallocate(ef->object);
ef->object = 0;
goto out;
}
/* Wire the pages */
error = vm_map_wire(kernel_map, mapbase,
mapbase + round_page(mapsize),
VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES);
if (error != KERN_SUCCESS) {
error = ENOMEM;
goto out;
}
/* Inform the kld system about the situation */
lf->address = ef->address = (caddr_t)mapbase;
lf->size = mapsize;
/*
* Now load code/data(progbits), zero bss(nobits), allocate space for
* and load relocs
*/
pb = 0;
rl = 0;
ra = 0;
alignmask = 0;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_size == 0)
continue;
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
alignmask = shdr[i].sh_addralign - 1;
mapbase += alignmask;
mapbase &= ~alignmask;
if (ef->shstrtab && shdr[i].sh_name != 0)
ef->progtab[pb].name =
ef->shstrtab + shdr[i].sh_name;
else if (shdr[i].sh_type == SHT_PROGBITS)
ef->progtab[pb].name = "<<PROGBITS>>";
else
ef->progtab[pb].name = "<<NOBITS>>";
if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, DPCPU_SETNAME))
ef->progtab[pb].addr =
dpcpu_alloc(shdr[i].sh_size);
#ifdef VIMAGE
else if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, VNET_SETNAME))
ef->progtab[pb].addr =
vnet_data_alloc(shdr[i].sh_size);
#endif
else
ef->progtab[pb].addr =
(void *)(uintptr_t)mapbase;
if (ef->progtab[pb].addr == NULL) {
error = ENOSPC;
goto out;
}
ef->progtab[pb].size = shdr[i].sh_size;
ef->progtab[pb].sec = i;
if (shdr[i].sh_type == SHT_PROGBITS) {
error = vn_rdwr(UIO_READ, nd.ni_vp,
ef->progtab[pb].addr,
shdr[i].sh_size, shdr[i].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred,
NOCRED, &resid, td);
if (error)
goto out;
if (resid != 0){
error = EINVAL;
goto out;
}
/* Initialize the per-cpu or vnet area. */
if (ef->progtab[pb].addr != (void *)mapbase &&
!strcmp(ef->progtab[pb].name, DPCPU_SETNAME))
dpcpu_copy(ef->progtab[pb].addr,
shdr[i].sh_size);
#ifdef VIMAGE
else if (ef->progtab[pb].addr !=
(void *)mapbase &&
!strcmp(ef->progtab[pb].name, VNET_SETNAME))
vnet_data_copy(ef->progtab[pb].addr,
shdr[i].sh_size);
#endif
} else
bzero(ef->progtab[pb].addr, shdr[i].sh_size);
/* Update all symbol values with the offset. */
for (j = 0; j < ef->ddbsymcnt; j++) {
es = &ef->ddbsymtab[j];
if (es->st_shndx != i)
continue;
es->st_value += (Elf_Addr)ef->progtab[pb].addr;
}
mapbase += shdr[i].sh_size;
pb++;
break;
case SHT_REL:
ef->reltab[rl].rel = malloc(shdr[i].sh_size, M_LINKER,
M_WAITOK);
ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel);
ef->reltab[rl].sec = shdr[i].sh_info;
error = vn_rdwr(UIO_READ, nd.ni_vp,
(void *)ef->reltab[rl].rel,
shdr[i].sh_size, shdr[i].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
&resid, td);
if (error)
goto out;
if (resid != 0){
error = EINVAL;
goto out;
}
rl++;
break;
case SHT_RELA:
ef->relatab[ra].rela = malloc(shdr[i].sh_size, M_LINKER,
M_WAITOK);
ef->relatab[ra].nrela =
shdr[i].sh_size / sizeof(Elf_Rela);
ef->relatab[ra].sec = shdr[i].sh_info;
error = vn_rdwr(UIO_READ, nd.ni_vp,
(void *)ef->relatab[ra].rela,
shdr[i].sh_size, shdr[i].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
&resid, td);
if (error)
goto out;
if (resid != 0){
error = EINVAL;
goto out;
}
ra++;
break;
}
}
if (pb != ef->nprogtab)
panic("lost progbits");
if (rl != ef->nreltab)
panic("lost reltab");
if (ra != ef->nrelatab)
panic("lost relatab");
if (mapbase != (vm_offset_t)ef->address + mapsize)
panic("mapbase 0x%lx != address %p + mapsize 0x%lx (0x%lx)\n",
(u_long)mapbase, ef->address, (u_long)mapsize,
(u_long)(vm_offset_t)ef->address + mapsize);
/* Local intra-module relocations */
link_elf_reloc_local(lf);
/* Pull in dependencies */
VOP_UNLOCK(nd.ni_vp, 0);
error = linker_load_dependencies(lf);
vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY);
if (error)
goto out;
/* External relocations */
error = relocate_file(ef);
if (error)
goto out;
/* Notify MD code that a module is being loaded. */
error = elf_cpu_load_file(lf);
if (error)
goto out;
*result = lf;
out:
VOP_UNLOCK(nd.ni_vp, 0);
vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
if (error && lf)
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
if (hdr)
free(hdr, M_LINKER);
return error;
}
static int
link_elf_link_preload(linker_class_t cls, const char *filename,
linker_file_t *result)
{
Elf_Ehdr *hdr;
Elf_Shdr *shdr;
Elf_Sym *es;
void *modptr, *baseptr, *sizeptr;
char *type;
elf_file_t ef;
linker_file_t lf;
Elf_Addr off;
int error, i, j, pb, ra, rl, shstrindex, symstrindex, symtabindex;
/* Look to see if we have the file preloaded */
modptr = preload_search_by_name(filename);
if (modptr == NULL)
return ENOENT;
type = (char *)preload_search_info(modptr, MODINFO_TYPE);
baseptr = preload_search_info(modptr, MODINFO_ADDR);
sizeptr = preload_search_info(modptr, MODINFO_SIZE);
hdr = (Elf_Ehdr *)preload_search_info(modptr, MODINFO_METADATA |
MODINFOMD_ELFHDR);
shdr = (Elf_Shdr *)preload_search_info(modptr, MODINFO_METADATA |
MODINFOMD_SHDR);
if (type == NULL || (strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE)
" obj module") != 0 &&
strcmp(type, "elf obj module") != 0)) {
return (EFTYPE);
}
if (baseptr == NULL || sizeptr == NULL || hdr == NULL ||
shdr == NULL)
return (EINVAL);
lf = linker_make_file(filename, &link_elf_class);
if (lf == NULL)
return (ENOMEM);
ef = (elf_file_t)lf;
ef->preloaded = 1;
ef->address = *(caddr_t *)baseptr;
lf->address = *(caddr_t *)baseptr;
lf->size = *(size_t *)sizeptr;
if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS ||
hdr->e_ident[EI_DATA] != ELF_TARG_DATA ||
hdr->e_ident[EI_VERSION] != EV_CURRENT ||
hdr->e_version != EV_CURRENT ||
hdr->e_type != ET_REL ||
hdr->e_machine != ELF_TARG_MACH) {
error = EFTYPE;
goto out;
}
ef->e_shdr = shdr;
/* Scan the section header for information and table sizing. */
symtabindex = -1;
symstrindex = -1;
for (i = 0; i < hdr->e_shnum; i++) {
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
ef->nprogtab++;
break;
case SHT_SYMTAB:
symtabindex = i;
symstrindex = shdr[i].sh_link;
break;
case SHT_REL:
ef->nreltab++;
break;
case SHT_RELA:
ef->nrelatab++;
break;
}
}
shstrindex = hdr->e_shstrndx;
if (ef->nprogtab == 0 || symstrindex < 0 ||
symstrindex >= hdr->e_shnum ||
shdr[symstrindex].sh_type != SHT_STRTAB || shstrindex == 0 ||
shstrindex >= hdr->e_shnum ||
shdr[shstrindex].sh_type != SHT_STRTAB) {
printf("%s: bad/missing section headers\n", filename);
error = ENOEXEC;
goto out;
}
/* Allocate space for tracking the load chunks */
if (ef->nprogtab != 0)
ef->progtab = malloc(ef->nprogtab * sizeof(*ef->progtab),
M_LINKER, M_WAITOK | M_ZERO);
if (ef->nreltab != 0)
ef->reltab = malloc(ef->nreltab * sizeof(*ef->reltab),
M_LINKER, M_WAITOK | M_ZERO);
if (ef->nrelatab != 0)
ef->relatab = malloc(ef->nrelatab * sizeof(*ef->relatab),
M_LINKER, M_WAITOK | M_ZERO);
if ((ef->nprogtab != 0 && ef->progtab == NULL) ||
(ef->nreltab != 0 && ef->reltab == NULL) ||
(ef->nrelatab != 0 && ef->relatab == NULL)) {
error = ENOMEM;
goto out;
}
/* XXX, relocate the sh_addr fields saved by the loader. */
off = 0;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_addr != 0 && (off == 0 || shdr[i].sh_addr < off))
off = shdr[i].sh_addr;
}
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_addr != 0)
shdr[i].sh_addr = shdr[i].sh_addr - off +
(Elf_Addr)ef->address;
}
ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym);
ef->ddbsymtab = (Elf_Sym *)shdr[symtabindex].sh_addr;
ef->ddbstrcnt = shdr[symstrindex].sh_size;
ef->ddbstrtab = (char *)shdr[symstrindex].sh_addr;
ef->shstrcnt = shdr[shstrindex].sh_size;
ef->shstrtab = (char *)shdr[shstrindex].sh_addr;
/* Now fill out progtab and the relocation tables. */
pb = 0;
rl = 0;
ra = 0;
for (i = 0; i < hdr->e_shnum; i++) {
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
ef->progtab[pb].addr = (void *)shdr[i].sh_addr;
if (shdr[i].sh_type == SHT_PROGBITS)
ef->progtab[pb].name = "<<PROGBITS>>";
else
ef->progtab[pb].name = "<<NOBITS>>";
ef->progtab[pb].size = shdr[i].sh_size;
ef->progtab[pb].sec = i;
if (ef->shstrtab && shdr[i].sh_name != 0)
ef->progtab[pb].name =
ef->shstrtab + shdr[i].sh_name;
if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) {
void *dpcpu;
dpcpu = dpcpu_alloc(shdr[i].sh_size);
if (dpcpu == NULL) {
error = ENOSPC;
goto out;
}
memcpy(dpcpu, ef->progtab[pb].addr,
ef->progtab[pb].size);
dpcpu_copy(dpcpu, shdr[i].sh_size);
ef->progtab[pb].addr = dpcpu;
#ifdef VIMAGE
} else if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, VNET_SETNAME)) {
void *vnet_data;
vnet_data = vnet_data_alloc(shdr[i].sh_size);
if (vnet_data == NULL) {
error = ENOSPC;
goto out;
}
memcpy(vnet_data, ef->progtab[pb].addr,
ef->progtab[pb].size);
vnet_data_copy(vnet_data, shdr[i].sh_size);
ef->progtab[pb].addr = vnet_data;
#endif
}
/* Update all symbol values with the offset. */
for (j = 0; j < ef->ddbsymcnt; j++) {
es = &ef->ddbsymtab[j];
if (es->st_shndx != i)
continue;
es->st_value += (Elf_Addr)ef->progtab[pb].addr;
}
pb++;
break;
case SHT_REL:
ef->reltab[rl].rel = (Elf_Rel *)shdr[i].sh_addr;
ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel);
ef->reltab[rl].sec = shdr[i].sh_info;
rl++;
break;
case SHT_RELA:
ef->relatab[ra].rela = (Elf_Rela *)shdr[i].sh_addr;
ef->relatab[ra].nrela =
shdr[i].sh_size / sizeof(Elf_Rela);
ef->relatab[ra].sec = shdr[i].sh_info;
ra++;
break;
}
}
if (pb != ef->nprogtab)
panic("lost progbits");
if (rl != ef->nreltab)
panic("lost reltab");
if (ra != ef->nrelatab)
panic("lost relatab");
/* Local intra-module relocations */
link_elf_reloc_local(lf);
*result = lf;
return (0);
out:
/* preload not done this way */
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
return (error);
}
static int
link_elf_load_file(const char* filename, linker_file_t* result)
{
struct nlookupdata nd;
struct thread *td = curthread; /* XXX */
struct proc *p = td->td_proc;
struct vnode *vp;
Elf_Ehdr *hdr;
caddr_t firstpage;
int nbytes, i;
Elf_Phdr *phdr;
Elf_Phdr *phlimit;
Elf_Phdr *segs[2];
int nsegs;
Elf_Phdr *phdyn;
caddr_t mapbase;
size_t mapsize;
Elf_Addr base_vaddr;
Elf_Addr base_vlimit;
int error = 0;
int resid;
elf_file_t ef;
linker_file_t lf;
char *pathname;
Elf_Shdr *shdr;
int symtabindex;
int symstrindex;
int symcnt;
int strcnt;
/* XXX Hack for firmware loading where p == NULL */
if (p == NULL) {
p = &proc0;
}
KKASSERT(p != NULL);
if (p->p_ucred == NULL) {
kprintf("link_elf_load_file: cannot load '%s' from filesystem"
" this early\n", filename);
return ENOENT;
}
shdr = NULL;
lf = NULL;
pathname = linker_search_path(filename);
if (pathname == NULL)
return ENOENT;
error = nlookup_init(&nd, pathname, UIO_SYSSPACE, NLC_FOLLOW|NLC_LOCKVP);
if (error == 0)
error = vn_open(&nd, NULL, FREAD, 0);
kfree(pathname, M_LINKER);
if (error) {
nlookup_done(&nd);
return error;
}
vp = nd.nl_open_vp;
nd.nl_open_vp = NULL;
nlookup_done(&nd);
/*
* Read the elf header from the file.
*/
firstpage = kmalloc(PAGE_SIZE, M_LINKER, M_WAITOK);
hdr = (Elf_Ehdr *)firstpage;
error = vn_rdwr(UIO_READ, vp, firstpage, PAGE_SIZE, 0,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
nbytes = PAGE_SIZE - resid;
if (error)
goto out;
if (!IS_ELF(*hdr)) {
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS
|| hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
link_elf_error("Unsupported file layout");
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_VERSION] != EV_CURRENT
|| hdr->e_version != EV_CURRENT) {
link_elf_error("Unsupported file version");
error = ENOEXEC;
goto out;
}
if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) {
error = ENOSYS;
goto out;
}
if (hdr->e_machine != ELF_TARG_MACH) {
link_elf_error("Unsupported machine");
error = ENOEXEC;
goto out;
}
/*
* We rely on the program header being in the first page. This is
* not strictly required by the ABI specification, but it seems to
* always true in practice. And, it simplifies things considerably.
*/
if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) &&
(hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) &&
(hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes)))
link_elf_error("Unreadable program headers");
/*
* Scan the program header entries, and save key information.
*
* We rely on there being exactly two load segments, text and data,
* in that order.
*/
phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff);
phlimit = phdr + hdr->e_phnum;
nsegs = 0;
phdyn = NULL;
while (phdr < phlimit) {
switch (phdr->p_type) {
case PT_LOAD:
if (nsegs == 2) {
link_elf_error("Too many sections");
error = ENOEXEC;
goto out;
}
segs[nsegs] = phdr;
++nsegs;
break;
case PT_PHDR:
break;
case PT_DYNAMIC:
phdyn = phdr;
break;
case PT_INTERP:
error = ENOSYS;
goto out;
}
++phdr;
}
if (phdyn == NULL) {
link_elf_error("Object is not dynamically-linked");
error = ENOEXEC;
goto out;
}
/*
* Allocate the entire address space of the object, to stake out our
* contiguous region, and to establish the base address for relocation.
*/
base_vaddr = trunc_page(segs[0]->p_vaddr);
base_vlimit = round_page(segs[1]->p_vaddr + segs[1]->p_memsz);
mapsize = base_vlimit - base_vaddr;
ef = kmalloc(sizeof(struct elf_file), M_LINKER, M_WAITOK | M_ZERO);
ef->address = kmalloc(mapsize, M_LINKER, M_WAITOK);
mapbase = ef->address;
/*
* Read the text and data sections and zero the bss.
*/
for (i = 0; i < 2; i++) {
caddr_t segbase = mapbase + segs[i]->p_vaddr - base_vaddr;
error = vn_rdwr(UIO_READ, vp,
segbase, segs[i]->p_filesz, segs[i]->p_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error) {
kfree(ef->address, M_LINKER);
kfree(ef, M_LINKER);
goto out;
}
bzero(segbase + segs[i]->p_filesz,
segs[i]->p_memsz - segs[i]->p_filesz);
}
ef->dynamic = (const Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr);
lf = linker_make_file(filename, ef, &link_elf_file_ops);
if (lf == NULL) {
kfree(ef->address, M_LINKER);
kfree(ef, M_LINKER);
error = ENOMEM;
goto out;
}
lf->address = ef->address;
lf->size = mapsize;
error = parse_dynamic(lf);
if (error)
goto out;
link_elf_reloc_local(lf);
error = linker_load_dependencies(lf);
if (error)
goto out;
error = relocate_file(lf);
if (error)
goto out;
/* Try and load the symbol table if it's present. (you can strip it!) */
nbytes = hdr->e_shnum * hdr->e_shentsize;
if (nbytes == 0 || hdr->e_shoff == 0)
goto nosyms;
shdr = kmalloc(nbytes, M_LINKER, M_WAITOK | M_ZERO);
error = vn_rdwr(UIO_READ, vp,
(caddr_t)shdr, nbytes, hdr->e_shoff,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
symtabindex = -1;
symstrindex = -1;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_type == SHT_SYMTAB) {
symtabindex = i;
symstrindex = shdr[i].sh_link;
}
}
if (symtabindex < 0 || symstrindex < 0)
goto nosyms;
symcnt = shdr[symtabindex].sh_size;
ef->symbase = kmalloc(symcnt, M_LINKER, M_WAITOK);
strcnt = shdr[symstrindex].sh_size;
ef->strbase = kmalloc(strcnt, M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, vp,
ef->symbase, symcnt, shdr[symtabindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
error = vn_rdwr(UIO_READ, vp,
ef->strbase, strcnt, shdr[symstrindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);
ef->ddbsymtab = (const Elf_Sym *)ef->symbase;
ef->ddbstrcnt = strcnt;
ef->ddbstrtab = ef->strbase;
nosyms:
*result = lf;
out:
if (error && lf)
linker_file_unload(lf);
if (shdr)
kfree(shdr, M_LINKER);
if (firstpage)
kfree(firstpage, M_LINKER);
vn_unlock(vp);
vn_close(vp, FREAD, NULL);
return error;
}
static int
link_elf_obj_load_file(const char *filename, linker_file_t * result)
{
struct nlookupdata nd;
struct thread *td = curthread; /* XXX */
struct proc *p = td->td_proc;
char *pathname;
struct vnode *vp;
Elf_Ehdr *hdr;
Elf_Shdr *shdr;
Elf_Sym *es;
int nbytes, i, j;
vm_offset_t mapbase;
size_t mapsize;
int error = 0;
int resid;
elf_file_t ef;
linker_file_t lf;
int symtabindex;
int symstrindex;
int shstrindex;
int nsym;
int pb, rl, ra;
int alignmask;
/* XXX Hack for firmware loading where p == NULL */
if (p == NULL) {
p = &proc0;
}
KKASSERT(p != NULL);
if (p->p_ucred == NULL) {
kprintf("link_elf_obj_load_file: cannot load '%s' from filesystem"
" this early\n", filename);
return ENOENT;
}
shdr = NULL;
lf = NULL;
mapsize = 0;
hdr = NULL;
pathname = linker_search_path(filename);
if (pathname == NULL)
return ENOENT;
error = nlookup_init(&nd, pathname, UIO_SYSSPACE, NLC_FOLLOW | NLC_LOCKVP);
if (error == 0)
error = vn_open(&nd, NULL, FREAD, 0);
kfree(pathname, M_LINKER);
if (error) {
nlookup_done(&nd);
return error;
}
vp = nd.nl_open_vp;
nd.nl_open_vp = NULL;
nlookup_done(&nd);
/*
* Read the elf header from the file.
*/
hdr = kmalloc(sizeof(*hdr), M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, vp, (void *)hdr, sizeof(*hdr), 0,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = ENOEXEC;
goto out;
}
if (!IS_ELF(*hdr)) {
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS
|| hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
link_elf_obj_error(filename, "Unsupported file layout");
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_VERSION] != EV_CURRENT
|| hdr->e_version != EV_CURRENT) {
link_elf_obj_error(filename, "Unsupported file version");
error = ENOEXEC;
goto out;
}
if (hdr->e_type != ET_REL) {
error = ENOSYS;
goto out;
}
if (hdr->e_machine != ELF_TARG_MACH) {
link_elf_obj_error(filename, "Unsupported machine");
error = ENOEXEC;
goto out;
}
ef = kmalloc(sizeof(struct elf_file), M_LINKER, M_WAITOK | M_ZERO);
lf = linker_make_file(filename, ef, &link_elf_obj_file_ops);
if (lf == NULL) {
kfree(ef, M_LINKER);
error = ENOMEM;
goto out;
}
ef->nprogtab = 0;
ef->e_shdr = NULL;
ef->nreltab = 0;
ef->nrelatab = 0;
/* Allocate and read in the section header */
nbytes = hdr->e_shnum * hdr->e_shentsize;
if (nbytes == 0 || hdr->e_shoff == 0 ||
hdr->e_shentsize != sizeof(Elf_Shdr)) {
error = ENOEXEC;
goto out;
}
shdr = kmalloc(nbytes, M_LINKER, M_WAITOK);
ef->e_shdr = shdr;
error = vn_rdwr(UIO_READ, vp, (caddr_t) shdr, nbytes, hdr->e_shoff,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid) {
error = ENOEXEC;
goto out;
}
/* Scan the section header for information and table sizing. */
nsym = 0;
symtabindex = -1;
symstrindex = -1;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_size == 0)
continue;
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
ef->nprogtab++;
break;
case SHT_SYMTAB:
nsym++;
symtabindex = i;
symstrindex = shdr[i].sh_link;
break;
case SHT_REL:
ef->nreltab++;
break;
case SHT_RELA:
ef->nrelatab++;
break;
case SHT_STRTAB:
break;
}
}
if (ef->nprogtab == 0) {
link_elf_obj_error(filename, "file has no contents");
error = ENOEXEC;
goto out;
}
if (nsym != 1) {
/* Only allow one symbol table for now */
link_elf_obj_error(filename, "file has no valid symbol table");
error = ENOEXEC;
goto out;
}
if (symstrindex < 0 || symstrindex > hdr->e_shnum ||
shdr[symstrindex].sh_type != SHT_STRTAB) {
link_elf_obj_error(filename, "file has invalid symbol strings");
error = ENOEXEC;
goto out;
}
/* Allocate space for tracking the load chunks */
if (ef->nprogtab != 0)
ef->progtab = kmalloc(ef->nprogtab * sizeof(*ef->progtab),
M_LINKER, M_WAITOK | M_ZERO);
if (ef->nreltab != 0)
ef->reltab = kmalloc(ef->nreltab * sizeof(*ef->reltab),
M_LINKER, M_WAITOK | M_ZERO);
if (ef->nrelatab != 0)
ef->relatab = kmalloc(ef->nrelatab * sizeof(*ef->relatab),
M_LINKER, M_WAITOK | M_ZERO);
if ((ef->nprogtab != 0 && ef->progtab == NULL) ||
(ef->nreltab != 0 && ef->reltab == NULL) ||
(ef->nrelatab != 0 && ef->relatab == NULL)) {
error = ENOMEM;
goto out;
}
if (symtabindex == -1)
panic("lost symbol table index");
/* Allocate space for and load the symbol table */
ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym);
ef->ddbsymtab = kmalloc(shdr[symtabindex].sh_size, M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, vp, (void *)ef->ddbsymtab,
shdr[symtabindex].sh_size, shdr[symtabindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
if (symstrindex == -1)
panic("lost symbol string index");
/* Allocate space for and load the symbol strings */
ef->ddbstrcnt = shdr[symstrindex].sh_size;
ef->ddbstrtab = kmalloc(shdr[symstrindex].sh_size, M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, vp, ef->ddbstrtab,
shdr[symstrindex].sh_size, shdr[symstrindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
/* Do we have a string table for the section names? */
shstrindex = -1;
if (hdr->e_shstrndx != 0 &&
shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) {
shstrindex = hdr->e_shstrndx;
ef->shstrcnt = shdr[shstrindex].sh_size;
ef->shstrtab = kmalloc(shdr[shstrindex].sh_size, M_LINKER,
M_WAITOK);
error = vn_rdwr(UIO_READ, vp, ef->shstrtab,
shdr[shstrindex].sh_size, shdr[shstrindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
}
/* Size up code/data(progbits) and bss(nobits). */
alignmask = 0;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_size == 0)
continue;
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
alignmask = shdr[i].sh_addralign - 1;
mapsize += alignmask;
mapsize &= ~alignmask;
mapsize += shdr[i].sh_size;
break;
}
}
/*
* We know how much space we need for the text/data/bss/etc. This
* stuff needs to be in a single chunk so that profiling etc can get
* the bounds and gdb can associate offsets with modules
*/
ef->object = vm_object_allocate(OBJT_DEFAULT,
round_page(mapsize) >> PAGE_SHIFT);
if (ef->object == NULL) {
error = ENOMEM;
goto out;
}
vm_object_hold(ef->object);
vm_object_reference_locked(ef->object);
ef->address = (caddr_t) vm_map_min(&kernel_map);
ef->bytes = 0;
/*
* In order to satisfy x86_64's architectural requirements on the
* location of code and data in the kernel's address space, request a
* mapping that is above the kernel.
*
* vkernel64's text+data is outside the managed VM space entirely.
*/
#if defined(__x86_64__) && defined(_KERNEL_VIRTUAL)
error = vkernel_module_memory_alloc(&mapbase, round_page(mapsize));
vm_object_drop(ef->object);
#else
mapbase = KERNBASE;
error = vm_map_find(&kernel_map, ef->object, NULL,
0, &mapbase, round_page(mapsize),
PAGE_SIZE,
TRUE, VM_MAPTYPE_NORMAL,
VM_PROT_ALL, VM_PROT_ALL, FALSE);
vm_object_drop(ef->object);
if (error) {
vm_object_deallocate(ef->object);
ef->object = NULL;
goto out;
}
/* Wire the pages */
error = vm_map_wire(&kernel_map, mapbase,
mapbase + round_page(mapsize), 0);
#endif
if (error != KERN_SUCCESS) {
error = ENOMEM;
goto out;
}
/* Inform the kld system about the situation */
lf->address = ef->address = (caddr_t) mapbase;
lf->size = round_page(mapsize);
ef->bytes = mapsize;
/*
* Now load code/data(progbits), zero bss(nobits), allocate space for
* and load relocs
*/
pb = 0;
rl = 0;
ra = 0;
alignmask = 0;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_size == 0)
continue;
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
alignmask = shdr[i].sh_addralign - 1;
mapbase += alignmask;
mapbase &= ~alignmask;
if (ef->shstrtab && shdr[i].sh_name != 0)
ef->progtab[pb].name =
ef->shstrtab + shdr[i].sh_name;
else if (shdr[i].sh_type == SHT_PROGBITS)
ef->progtab[pb].name = "<<PROGBITS>>";
else
ef->progtab[pb].name = "<<NOBITS>>";
#if 0
if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, "set_pcpu"))
ef->progtab[pb].addr =
dpcpu_alloc(shdr[i].sh_size);
#ifdef VIMAGE
else if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, VNET_SETNAME))
ef->progtab[pb].addr =
vnet_data_alloc(shdr[i].sh_size);
#endif
else
#endif
ef->progtab[pb].addr =
(void *)(uintptr_t) mapbase;
if (ef->progtab[pb].addr == NULL) {
error = ENOSPC;
goto out;
}
ef->progtab[pb].size = shdr[i].sh_size;
ef->progtab[pb].sec = i;
if (shdr[i].sh_type == SHT_PROGBITS) {
error = vn_rdwr(UIO_READ, vp,
ef->progtab[pb].addr,
shdr[i].sh_size, shdr[i].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred,
&resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
#if 0
/* Initialize the per-cpu or vnet area. */
if (ef->progtab[pb].addr != (void *)mapbase &&
!strcmp(ef->progtab[pb].name, "set_pcpu"))
dpcpu_copy(ef->progtab[pb].addr,
shdr[i].sh_size);
#ifdef VIMAGE
else if (ef->progtab[pb].addr !=
(void *)mapbase &&
!strcmp(ef->progtab[pb].name, VNET_SETNAME))
vnet_data_copy(ef->progtab[pb].addr,
shdr[i].sh_size);
#endif
#endif
} else
bzero(ef->progtab[pb].addr, shdr[i].sh_size);
/* Update all symbol values with the offset. */
for (j = 0; j < ef->ddbsymcnt; j++) {
es = &ef->ddbsymtab[j];
if (es->st_shndx != i)
continue;
es->st_value += (Elf_Addr) ef->progtab[pb].addr;
}
mapbase += shdr[i].sh_size;
pb++;
break;
case SHT_REL:
ef->reltab[rl].rel = kmalloc(shdr[i].sh_size, M_LINKER, M_WAITOK);
ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel);
ef->reltab[rl].sec = shdr[i].sh_info;
error = vn_rdwr(UIO_READ, vp,
(void *)ef->reltab[rl].rel,
shdr[i].sh_size, shdr[i].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
rl++;
break;
case SHT_RELA:
ef->relatab[ra].rela = kmalloc(shdr[i].sh_size, M_LINKER, M_WAITOK);
ef->relatab[ra].nrela = shdr[i].sh_size / sizeof(Elf_Rela);
ef->relatab[ra].sec = shdr[i].sh_info;
error = vn_rdwr(UIO_READ, vp,
(void *)ef->relatab[ra].rela,
shdr[i].sh_size, shdr[i].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
ra++;
break;
}
}
if (pb != ef->nprogtab)
panic("lost progbits");
if (rl != ef->nreltab)
panic("lost reltab");
if (ra != ef->nrelatab)
panic("lost relatab");
if (mapbase != (vm_offset_t) ef->address + mapsize)
panic("mapbase 0x%lx != address %p + mapsize 0x%lx (0x%lx)",
mapbase, ef->address, mapsize,
(vm_offset_t) ef->address + mapsize);
/* Local intra-module relocations */
link_elf_obj_reloc_local(lf);
/* Pull in dependencies */
error = linker_load_dependencies(lf);
if (error)
goto out;
/* External relocations */
error = relocate_file(lf);
if (error)
goto out;
*result = lf;
out:
if (error && lf)
linker_file_unload(lf /*, LINKER_UNLOAD_FORCE */);
if (hdr)
kfree(hdr, M_LINKER);
vn_unlock(vp);
vn_close(vp, FREAD, NULL);
return error;
}
