static int
link_elf_load_file(const char* filename, linker_file_t* result)
{
struct nlookupdata nd;
struct thread *td = curthread; /* XXX */
struct proc *p = td->td_proc;
struct vnode *vp;
Elf_Ehdr *hdr;
caddr_t firstpage;
int nbytes, i;
Elf_Phdr *phdr;
Elf_Phdr *phlimit;
Elf_Phdr *segs[2];
int nsegs;
Elf_Phdr *phdyn;
caddr_t mapbase;
size_t mapsize;
Elf_Addr base_vaddr;
Elf_Addr base_vlimit;
int error = 0;
int resid;
elf_file_t ef;
linker_file_t lf;
char *pathname;
Elf_Shdr *shdr;
int symtabindex;
int symstrindex;
int symcnt;
int strcnt;
/* XXX Hack for firmware loading where p == NULL */
if (p == NULL) {
p = &proc0;
}
KKASSERT(p != NULL);
if (p->p_ucred == NULL) {
kprintf("link_elf_load_file: cannot load '%s' from filesystem"
" this early\n", filename);
return ENOENT;
}
shdr = NULL;
lf = NULL;
pathname = linker_search_path(filename);
if (pathname == NULL)
return ENOENT;
error = nlookup_init(&nd, pathname, UIO_SYSSPACE, NLC_FOLLOW|NLC_LOCKVP);
if (error == 0)
error = vn_open(&nd, NULL, FREAD, 0);
kfree(pathname, M_LINKER);
if (error) {
nlookup_done(&nd);
return error;
}
vp = nd.nl_open_vp;
nd.nl_open_vp = NULL;
nlookup_done(&nd);
/*
* Read the elf header from the file.
*/
firstpage = kmalloc(PAGE_SIZE, M_LINKER, M_WAITOK);
hdr = (Elf_Ehdr *)firstpage;
error = vn_rdwr(UIO_READ, vp, firstpage, PAGE_SIZE, 0,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
nbytes = PAGE_SIZE - resid;
if (error)
goto out;
if (!IS_ELF(*hdr)) {
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS
|| hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
link_elf_error("Unsupported file layout");
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_VERSION] != EV_CURRENT
|| hdr->e_version != EV_CURRENT) {
link_elf_error("Unsupported file version");
error = ENOEXEC;
goto out;
}
if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) {
error = ENOSYS;
goto out;
}
if (hdr->e_machine != ELF_TARG_MACH) {
link_elf_error("Unsupported machine");
error = ENOEXEC;
goto out;
}
/*
* We rely on the program header being in the first page. This is
* not strictly required by the ABI specification, but it seems to
* always true in practice. And, it simplifies things considerably.
*/
if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) &&
(hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) &&
(hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes)))
link_elf_error("Unreadable program headers");
/*
* Scan the program header entries, and save key information.
*
* We rely on there being exactly two load segments, text and data,
* in that order.
*/
phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff);
phlimit = phdr + hdr->e_phnum;
nsegs = 0;
phdyn = NULL;
while (phdr < phlimit) {
switch (phdr->p_type) {
case PT_LOAD:
if (nsegs == 2) {
link_elf_error("Too many sections");
error = ENOEXEC;
goto out;
}
segs[nsegs] = phdr;
++nsegs;
break;
case PT_PHDR:
break;
case PT_DYNAMIC:
phdyn = phdr;
break;
case PT_INTERP:
error = ENOSYS;
goto out;
}
++phdr;
}
if (phdyn == NULL) {
link_elf_error("Object is not dynamically-linked");
error = ENOEXEC;
goto out;
}
/*
* Allocate the entire address space of the object, to stake out our
* contiguous region, and to establish the base address for relocation.
*/
base_vaddr = trunc_page(segs[0]->p_vaddr);
base_vlimit = round_page(segs[1]->p_vaddr + segs[1]->p_memsz);
mapsize = base_vlimit - base_vaddr;
ef = kmalloc(sizeof(struct elf_file), M_LINKER, M_WAITOK | M_ZERO);
ef->address = kmalloc(mapsize, M_LINKER, M_WAITOK);
mapbase = ef->address;
/*
* Read the text and data sections and zero the bss.
*/
for (i = 0; i < 2; i++) {
caddr_t segbase = mapbase + segs[i]->p_vaddr - base_vaddr;
error = vn_rdwr(UIO_READ, vp,
segbase, segs[i]->p_filesz, segs[i]->p_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error) {
kfree(ef->address, M_LINKER);
kfree(ef, M_LINKER);
goto out;
}
bzero(segbase + segs[i]->p_filesz,
segs[i]->p_memsz - segs[i]->p_filesz);
}
ef->dynamic = (const Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr);
lf = linker_make_file(filename, ef, &link_elf_file_ops);
if (lf == NULL) {
kfree(ef->address, M_LINKER);
kfree(ef, M_LINKER);
error = ENOMEM;
goto out;
}
lf->address = ef->address;
lf->size = mapsize;
error = parse_dynamic(lf);
if (error)
goto out;
link_elf_reloc_local(lf);
error = linker_load_dependencies(lf);
if (error)
goto out;
error = relocate_file(lf);
if (error)
goto out;
/* Try and load the symbol table if it's present. (you can strip it!) */
nbytes = hdr->e_shnum * hdr->e_shentsize;
if (nbytes == 0 || hdr->e_shoff == 0)
goto nosyms;
shdr = kmalloc(nbytes, M_LINKER, M_WAITOK | M_ZERO);
error = vn_rdwr(UIO_READ, vp,
(caddr_t)shdr, nbytes, hdr->e_shoff,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
symtabindex = -1;
symstrindex = -1;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_type == SHT_SYMTAB) {
symtabindex = i;
symstrindex = shdr[i].sh_link;
}
}
if (symtabindex < 0 || symstrindex < 0)
goto nosyms;
symcnt = shdr[symtabindex].sh_size;
ef->symbase = kmalloc(symcnt, M_LINKER, M_WAITOK);
strcnt = shdr[symstrindex].sh_size;
ef->strbase = kmalloc(strcnt, M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, vp,
ef->symbase, symcnt, shdr[symtabindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
error = vn_rdwr(UIO_READ, vp,
ef->strbase, strcnt, shdr[symstrindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);
ef->ddbsymtab = (const Elf_Sym *)ef->symbase;
ef->ddbstrcnt = strcnt;
ef->ddbstrtab = ef->strbase;
nosyms:
*result = lf;
out:
if (error && lf)
linker_file_unload(lf);
if (shdr)
kfree(shdr, M_LINKER);
if (firstpage)
kfree(firstpage, M_LINKER);
vn_unlock(vp);
vn_close(vp, FREAD, NULL);
return error;
}
static int
link_elf_obj_load_file(const char *filename, linker_file_t * result)
{
struct nlookupdata nd;
struct thread *td = curthread; /* XXX */
struct proc *p = td->td_proc;
char *pathname;
struct vnode *vp;
Elf_Ehdr *hdr;
Elf_Shdr *shdr;
Elf_Sym *es;
int nbytes, i, j;
vm_offset_t mapbase;
size_t mapsize;
int error = 0;
int resid;
elf_file_t ef;
linker_file_t lf;
int symtabindex;
int symstrindex;
int shstrindex;
int nsym;
int pb, rl, ra;
int alignmask;
/* XXX Hack for firmware loading where p == NULL */
if (p == NULL) {
p = &proc0;
}
KKASSERT(p != NULL);
if (p->p_ucred == NULL) {
kprintf("link_elf_obj_load_file: cannot load '%s' from filesystem"
" this early\n", filename);
return ENOENT;
}
shdr = NULL;
lf = NULL;
mapsize = 0;
hdr = NULL;
pathname = linker_search_path(filename);
if (pathname == NULL)
return ENOENT;
error = nlookup_init(&nd, pathname, UIO_SYSSPACE, NLC_FOLLOW | NLC_LOCKVP);
if (error == 0)
error = vn_open(&nd, NULL, FREAD, 0);
kfree(pathname, M_LINKER);
if (error) {
nlookup_done(&nd);
return error;
}
vp = nd.nl_open_vp;
nd.nl_open_vp = NULL;
nlookup_done(&nd);
/*
* Read the elf header from the file.
*/
hdr = kmalloc(sizeof(*hdr), M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, vp, (void *)hdr, sizeof(*hdr), 0,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = ENOEXEC;
goto out;
}
if (!IS_ELF(*hdr)) {
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS
|| hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
link_elf_obj_error(filename, "Unsupported file layout");
error = ENOEXEC;
goto out;
}
if (hdr->e_ident[EI_VERSION] != EV_CURRENT
|| hdr->e_version != EV_CURRENT) {
link_elf_obj_error(filename, "Unsupported file version");
error = ENOEXEC;
goto out;
}
if (hdr->e_type != ET_REL) {
error = ENOSYS;
goto out;
}
if (hdr->e_machine != ELF_TARG_MACH) {
link_elf_obj_error(filename, "Unsupported machine");
error = ENOEXEC;
goto out;
}
ef = kmalloc(sizeof(struct elf_file), M_LINKER, M_WAITOK | M_ZERO);
lf = linker_make_file(filename, ef, &link_elf_obj_file_ops);
if (lf == NULL) {
kfree(ef, M_LINKER);
error = ENOMEM;
goto out;
}
ef->nprogtab = 0;
ef->e_shdr = NULL;
ef->nreltab = 0;
ef->nrelatab = 0;
/* Allocate and read in the section header */
nbytes = hdr->e_shnum * hdr->e_shentsize;
if (nbytes == 0 || hdr->e_shoff == 0 ||
hdr->e_shentsize != sizeof(Elf_Shdr)) {
error = ENOEXEC;
goto out;
}
shdr = kmalloc(nbytes, M_LINKER, M_WAITOK);
ef->e_shdr = shdr;
error = vn_rdwr(UIO_READ, vp, (caddr_t) shdr, nbytes, hdr->e_shoff,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid) {
error = ENOEXEC;
goto out;
}
/* Scan the section header for information and table sizing. */
nsym = 0;
symtabindex = -1;
symstrindex = -1;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_size == 0)
continue;
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
ef->nprogtab++;
break;
case SHT_SYMTAB:
nsym++;
symtabindex = i;
symstrindex = shdr[i].sh_link;
break;
case SHT_REL:
ef->nreltab++;
break;
case SHT_RELA:
ef->nrelatab++;
break;
case SHT_STRTAB:
break;
}
}
if (ef->nprogtab == 0) {
link_elf_obj_error(filename, "file has no contents");
error = ENOEXEC;
goto out;
}
if (nsym != 1) {
/* Only allow one symbol table for now */
link_elf_obj_error(filename, "file has no valid symbol table");
error = ENOEXEC;
goto out;
}
if (symstrindex < 0 || symstrindex > hdr->e_shnum ||
shdr[symstrindex].sh_type != SHT_STRTAB) {
link_elf_obj_error(filename, "file has invalid symbol strings");
error = ENOEXEC;
goto out;
}
/* Allocate space for tracking the load chunks */
if (ef->nprogtab != 0)
ef->progtab = kmalloc(ef->nprogtab * sizeof(*ef->progtab),
M_LINKER, M_WAITOK | M_ZERO);
if (ef->nreltab != 0)
ef->reltab = kmalloc(ef->nreltab * sizeof(*ef->reltab),
M_LINKER, M_WAITOK | M_ZERO);
if (ef->nrelatab != 0)
ef->relatab = kmalloc(ef->nrelatab * sizeof(*ef->relatab),
M_LINKER, M_WAITOK | M_ZERO);
if ((ef->nprogtab != 0 && ef->progtab == NULL) ||
(ef->nreltab != 0 && ef->reltab == NULL) ||
(ef->nrelatab != 0 && ef->relatab == NULL)) {
error = ENOMEM;
goto out;
}
if (symtabindex == -1)
panic("lost symbol table index");
/* Allocate space for and load the symbol table */
ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym);
ef->ddbsymtab = kmalloc(shdr[symtabindex].sh_size, M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, vp, (void *)ef->ddbsymtab,
shdr[symtabindex].sh_size, shdr[symtabindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
if (symstrindex == -1)
panic("lost symbol string index");
/* Allocate space for and load the symbol strings */
ef->ddbstrcnt = shdr[symstrindex].sh_size;
ef->ddbstrtab = kmalloc(shdr[symstrindex].sh_size, M_LINKER, M_WAITOK);
error = vn_rdwr(UIO_READ, vp, ef->ddbstrtab,
shdr[symstrindex].sh_size, shdr[symstrindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
/* Do we have a string table for the section names? */
shstrindex = -1;
if (hdr->e_shstrndx != 0 &&
shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) {
shstrindex = hdr->e_shstrndx;
ef->shstrcnt = shdr[shstrindex].sh_size;
ef->shstrtab = kmalloc(shdr[shstrindex].sh_size, M_LINKER,
M_WAITOK);
error = vn_rdwr(UIO_READ, vp, ef->shstrtab,
shdr[shstrindex].sh_size, shdr[shstrindex].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
}
/* Size up code/data(progbits) and bss(nobits). */
alignmask = 0;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_size == 0)
continue;
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
alignmask = shdr[i].sh_addralign - 1;
mapsize += alignmask;
mapsize &= ~alignmask;
mapsize += shdr[i].sh_size;
break;
}
}
/*
* We know how much space we need for the text/data/bss/etc. This
* stuff needs to be in a single chunk so that profiling etc can get
* the bounds and gdb can associate offsets with modules
*/
ef->object = vm_object_allocate(OBJT_DEFAULT,
round_page(mapsize) >> PAGE_SHIFT);
if (ef->object == NULL) {
error = ENOMEM;
goto out;
}
vm_object_hold(ef->object);
vm_object_reference_locked(ef->object);
ef->address = (caddr_t) vm_map_min(&kernel_map);
ef->bytes = 0;
/*
* In order to satisfy x86_64's architectural requirements on the
* location of code and data in the kernel's address space, request a
* mapping that is above the kernel.
*
* vkernel64's text+data is outside the managed VM space entirely.
*/
#if defined(__x86_64__) && defined(_KERNEL_VIRTUAL)
error = vkernel_module_memory_alloc(&mapbase, round_page(mapsize));
vm_object_drop(ef->object);
#else
mapbase = KERNBASE;
error = vm_map_find(&kernel_map, ef->object, NULL,
0, &mapbase, round_page(mapsize),
PAGE_SIZE,
TRUE, VM_MAPTYPE_NORMAL,
VM_PROT_ALL, VM_PROT_ALL, FALSE);
vm_object_drop(ef->object);
if (error) {
vm_object_deallocate(ef->object);
ef->object = NULL;
goto out;
}
/* Wire the pages */
error = vm_map_wire(&kernel_map, mapbase,
mapbase + round_page(mapsize), 0);
#endif
if (error != KERN_SUCCESS) {
error = ENOMEM;
goto out;
}
/* Inform the kld system about the situation */
lf->address = ef->address = (caddr_t) mapbase;
lf->size = round_page(mapsize);
ef->bytes = mapsize;
/*
* Now load code/data(progbits), zero bss(nobits), allocate space for
* and load relocs
*/
pb = 0;
rl = 0;
ra = 0;
alignmask = 0;
for (i = 0; i < hdr->e_shnum; i++) {
if (shdr[i].sh_size == 0)
continue;
switch (shdr[i].sh_type) {
case SHT_PROGBITS:
case SHT_NOBITS:
alignmask = shdr[i].sh_addralign - 1;
mapbase += alignmask;
mapbase &= ~alignmask;
if (ef->shstrtab && shdr[i].sh_name != 0)
ef->progtab[pb].name =
ef->shstrtab + shdr[i].sh_name;
else if (shdr[i].sh_type == SHT_PROGBITS)
ef->progtab[pb].name = "<<PROGBITS>>";
else
ef->progtab[pb].name = "<<NOBITS>>";
#if 0
if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, "set_pcpu"))
ef->progtab[pb].addr =
dpcpu_alloc(shdr[i].sh_size);
#ifdef VIMAGE
else if (ef->progtab[pb].name != NULL &&
!strcmp(ef->progtab[pb].name, VNET_SETNAME))
ef->progtab[pb].addr =
vnet_data_alloc(shdr[i].sh_size);
#endif
else
#endif
ef->progtab[pb].addr =
(void *)(uintptr_t) mapbase;
if (ef->progtab[pb].addr == NULL) {
error = ENOSPC;
goto out;
}
ef->progtab[pb].size = shdr[i].sh_size;
ef->progtab[pb].sec = i;
if (shdr[i].sh_type == SHT_PROGBITS) {
error = vn_rdwr(UIO_READ, vp,
ef->progtab[pb].addr,
shdr[i].sh_size, shdr[i].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred,
&resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
#if 0
/* Initialize the per-cpu or vnet area. */
if (ef->progtab[pb].addr != (void *)mapbase &&
!strcmp(ef->progtab[pb].name, "set_pcpu"))
dpcpu_copy(ef->progtab[pb].addr,
shdr[i].sh_size);
#ifdef VIMAGE
else if (ef->progtab[pb].addr !=
(void *)mapbase &&
!strcmp(ef->progtab[pb].name, VNET_SETNAME))
vnet_data_copy(ef->progtab[pb].addr,
shdr[i].sh_size);
#endif
#endif
} else
bzero(ef->progtab[pb].addr, shdr[i].sh_size);
/* Update all symbol values with the offset. */
for (j = 0; j < ef->ddbsymcnt; j++) {
es = &ef->ddbsymtab[j];
if (es->st_shndx != i)
continue;
es->st_value += (Elf_Addr) ef->progtab[pb].addr;
}
mapbase += shdr[i].sh_size;
pb++;
break;
case SHT_REL:
ef->reltab[rl].rel = kmalloc(shdr[i].sh_size, M_LINKER, M_WAITOK);
ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel);
ef->reltab[rl].sec = shdr[i].sh_info;
error = vn_rdwr(UIO_READ, vp,
(void *)ef->reltab[rl].rel,
shdr[i].sh_size, shdr[i].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
rl++;
break;
case SHT_RELA:
ef->relatab[ra].rela = kmalloc(shdr[i].sh_size, M_LINKER, M_WAITOK);
ef->relatab[ra].nrela = shdr[i].sh_size / sizeof(Elf_Rela);
ef->relatab[ra].sec = shdr[i].sh_info;
error = vn_rdwr(UIO_READ, vp,
(void *)ef->relatab[ra].rela,
shdr[i].sh_size, shdr[i].sh_offset,
UIO_SYSSPACE, IO_NODELOCKED, p->p_ucred, &resid);
if (error)
goto out;
if (resid != 0) {
error = EINVAL;
goto out;
}
ra++;
break;
}
}
if (pb != ef->nprogtab)
panic("lost progbits");
if (rl != ef->nreltab)
panic("lost reltab");
if (ra != ef->nrelatab)
panic("lost relatab");
if (mapbase != (vm_offset_t) ef->address + mapsize)
panic("mapbase 0x%lx != address %p + mapsize 0x%lx (0x%lx)",
mapbase, ef->address, mapsize,
(vm_offset_t) ef->address + mapsize);
/* Local intra-module relocations */
link_elf_obj_reloc_local(lf);
/* Pull in dependencies */
error = linker_load_dependencies(lf);
if (error)
goto out;
/* External relocations */
error = relocate_file(lf);
if (error)
goto out;
*result = lf;
out:
if (error && lf)
linker_file_unload(lf /*, LINKER_UNLOAD_FORCE */);
if (hdr)
kfree(hdr, M_LINKER);
vn_unlock(vp);
vn_close(vp, FREAD, NULL);
return error;
}
