/*
* Routine: macx_swapon
* Function:
* Syscall interface to add a file to backing store
*/
int
macx_swapon(
struct macx_swapon_args *args)
{
int size = args->size;
vnode_t vp = (vnode_t)NULL;
struct nameidata nd, *ndp;
register int error;
kern_return_t kr;
mach_port_t backing_store;
memory_object_default_t default_pager;
int i;
boolean_t funnel_state;
off_t file_size;
vfs_context_t ctx = vfs_context_current();
struct proc *p = current_proc();
int dp_cluster_size;
AUDIT_MACH_SYSCALL_ENTER(AUE_SWAPON);
AUDIT_ARG(value32, args->priority);
funnel_state = thread_funnel_set(kernel_flock, TRUE);
ndp = &nd;
if ((error = suser(kauth_cred_get(), 0)))
goto swapon_bailout;
/*
* Get a vnode for the paging area.
*/
NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1,
((IS_64BIT_PROCESS(p)) ? UIO_USERSPACE64 : UIO_USERSPACE32),
(user_addr_t) args->filename, ctx);
if ((error = namei(ndp)))
goto swapon_bailout;
nameidone(ndp);
vp = ndp->ni_vp;
if (vp->v_type != VREG) {
error = EINVAL;
goto swapon_bailout;
}
/* get file size */
if ((error = vnode_size(vp, &file_size, ctx)) != 0)
goto swapon_bailout;
#if CONFIG_MACF
vnode_lock(vp);
error = mac_system_check_swapon(vfs_context_ucred(ctx), vp);
vnode_unlock(vp);
if (error)
goto swapon_bailout;
#endif
/* resize to desired size if it's too small */
if ((file_size < (off_t)size) && ((error = vnode_setsize(vp, (off_t)size, 0, ctx)) != 0))
goto swapon_bailout;
if (default_pager_init_flag == 0) {
start_def_pager(NULL);
default_pager_init_flag = 1;
}
/* add new backing store to list */
i = 0;
while(bs_port_table[i].vp != 0) {
if(i == MAX_BACKING_STORE)
break;
i++;
}
if(i == MAX_BACKING_STORE) {
error = ENOMEM;
goto swapon_bailout;
}
/* remember the vnode. This vnode has namei() reference */
bs_port_table[i].vp = vp;
/*
* Look to see if we are already paging to this file.
*/
/* make certain the copy send of kernel call will work */
default_pager = MEMORY_OBJECT_DEFAULT_NULL;
kr = host_default_memory_manager(host_priv_self(), &default_pager, 0);
if(kr != KERN_SUCCESS) {
error = EAGAIN;
bs_port_table[i].vp = 0;
goto swapon_bailout;
}
if (vp->v_mount->mnt_kern_flag & MNTK_SSD) {
/*
* keep the cluster size small since the
* seek cost is effectively 0 which means
* we don't care much about fragmentation
*/
dp_isssd = TRUE;
dp_cluster_size = 2 * PAGE_SIZE;
} else {
/*
* use the default cluster size
*/
dp_isssd = FALSE;
dp_cluster_size = 0;
}
kr = default_pager_backing_store_create(default_pager,
-1, /* default priority */
dp_cluster_size,
&backing_store);
memory_object_default_deallocate(default_pager);
if(kr != KERN_SUCCESS) {
error = ENOMEM;
bs_port_table[i].vp = 0;
goto swapon_bailout;
}
/* Mark this vnode as being used for swapfile */
vnode_lock_spin(vp);
SET(vp->v_flag, VSWAP);
vnode_unlock(vp);
/*
* NOTE: we are able to supply PAGE_SIZE here instead of
* an actual record size or block number because:
* a: we do not support offsets from the beginning of the
* file (allowing for non page size/record modulo offsets.
* b: because allow paging will be done modulo page size
*/
kr = default_pager_add_file(backing_store, (vnode_ptr_t) vp,
PAGE_SIZE, (int)(file_size/PAGE_SIZE));
if(kr != KERN_SUCCESS) {
bs_port_table[i].vp = 0;
if(kr == KERN_INVALID_ARGUMENT)
error = EINVAL;
else
error = ENOMEM;
/* This vnode is not to be used for swapfile */
vnode_lock_spin(vp);
CLR(vp->v_flag, VSWAP);
vnode_unlock(vp);
goto swapon_bailout;
}
bs_port_table[i].bs = (void *)backing_store;
error = 0;
ubc_setthreadcred(vp, p, current_thread());
/*
* take a long term reference on the vnode to keep
* vnreclaim() away from this vnode.
*/
vnode_ref(vp);
swapon_bailout:
if (vp) {
vnode_put(vp);
}
(void) thread_funnel_set(kernel_flock, FALSE);
AUDIT_MACH_SYSCALL_EXIT(error);
return(error);
}
/*
* Routine: macx_swapinfo
* Function:
* Syscall interface to get general swap statistics
*/
int
macx_swapinfo(
memory_object_size_t *total_p,
memory_object_size_t *avail_p,
vm_size_t *pagesize_p,
boolean_t *encrypted_p)
{
int error;
memory_object_default_t default_pager;
default_pager_info_64_t dpi64;
kern_return_t kr;
error = 0;
/*
* Get a handle on the default pager.
*/
default_pager = MEMORY_OBJECT_DEFAULT_NULL;
kr = host_default_memory_manager(host_priv_self(), &default_pager, 0);
if (kr != KERN_SUCCESS) {
error = EAGAIN; /* XXX why EAGAIN ? */
goto done;
}
if (default_pager == MEMORY_OBJECT_DEFAULT_NULL) {
/*
* The default pager has not initialized yet,
* so it can't be using any swap space at all.
*/
*total_p = 0;
*avail_p = 0;
*pagesize_p = 0;
*encrypted_p = FALSE;
goto done;
}
/*
* Get swap usage data from default pager.
*/
kr = default_pager_info_64(default_pager, &dpi64);
if (kr != KERN_SUCCESS) {
error = ENOTSUP;
goto done;
}
/*
* Provide default pager info to caller.
*/
*total_p = dpi64.dpi_total_space;
*avail_p = dpi64.dpi_free_space;
*pagesize_p = dpi64.dpi_page_size;
if (dpi64.dpi_flags & DPI_ENCRYPTED) {
*encrypted_p = TRUE;
} else {
*encrypted_p = FALSE;
}
done:
if (default_pager != MEMORY_OBJECT_DEFAULT_NULL) {
/* release our handle on default pager */
memory_object_default_deallocate(default_pager);
}
return error;
}
void
default_pager_initialize(void)
{
kern_return_t kr;
__unused static char here[] = "default_pager_initialize";
lck_grp_attr_setdefault(&default_pager_lck_grp_attr);
lck_grp_init(&default_pager_lck_grp, "default_pager", &default_pager_lck_grp_attr);
lck_attr_setdefault(&default_pager_lck_attr);
/*
* Vm variables.
*/
#ifndef MACH_KERNEL
vm_page_mask = vm_page_size - 1;
assert((unsigned int) vm_page_size == vm_page_size);
vm_page_shift = local_log2((unsigned int) vm_page_size);
#endif
/*
* List of all vstructs.
*/
vstruct_zone = zinit(sizeof(struct vstruct),
10000 * sizeof(struct vstruct),
8192, "vstruct zone");
zone_change(vstruct_zone, Z_CALLERACCT, FALSE);
zone_change(vstruct_zone, Z_NOENCRYPT, TRUE);
VSL_LOCK_INIT();
queue_init(&vstruct_list.vsl_queue);
vstruct_list.vsl_count = 0;
VSTATS_LOCK_INIT(&global_stats.gs_lock);
bs_initialize();
/*
* Exported DMM port.
*/
default_pager_object = ipc_port_alloc_kernel();
/*
* Export pager interfaces.
*/
#ifdef USER_PAGER
if ((kr = netname_check_in(name_server_port, "UserPager",
default_pager_self,
default_pager_object))
!= KERN_SUCCESS) {
dprintf(("netname_check_in returned 0x%x\n", kr));
exit(1);
}
#else /* USER_PAGER */
{
unsigned int clsize;
memory_object_default_t dmm;
dmm = default_pager_object;
assert((unsigned int) vm_page_size == vm_page_size);
clsize = ((unsigned int) vm_page_size << vstruct_def_clshift);
kr = host_default_memory_manager(host_priv_self(), &dmm, clsize);
if ((kr != KERN_SUCCESS) ||
(dmm != MEMORY_OBJECT_DEFAULT_NULL))
Panic("default memory manager");
}
#endif /* USER_PAGER */
}
/*
* Routine: macx_triggers
* Function:
* Syscall interface to set the call backs for low and
* high water marks.
*/
int
macx_triggers(
struct macx_triggers_args *args)
{
int hi_water = args->hi_water;
int low_water = args->low_water;
int flags = args->flags;
mach_port_t trigger_name = args->alert_port;
kern_return_t kr;
memory_object_default_t default_pager;
ipc_port_t trigger_port;
default_pager = MEMORY_OBJECT_DEFAULT_NULL;
kr = host_default_memory_manager(host_priv_self(),
&default_pager, 0);
if(kr != KERN_SUCCESS) {
return EINVAL;
}
if ((flags & SWAP_ENCRYPT_ON) &&
(flags & SWAP_ENCRYPT_OFF)) {
/* can't have it both ways */
return EINVAL;
}
if (default_pager_init_flag == 0) {
start_def_pager(NULL);
default_pager_init_flag = 1;
}
if (flags & SWAP_ENCRYPT_ON) {
/* ENCRYPTED SWAP: tell default_pager to encrypt */
default_pager_triggers(default_pager,
0, 0,
SWAP_ENCRYPT_ON,
IP_NULL);
} else if (flags & SWAP_ENCRYPT_OFF) {
/* ENCRYPTED SWAP: tell default_pager not to encrypt */
default_pager_triggers(default_pager,
0, 0,
SWAP_ENCRYPT_OFF,
IP_NULL);
}
if (flags & HI_WAT_ALERT) {
trigger_port = trigger_name_to_port(trigger_name);
if(trigger_port == NULL) {
return EINVAL;
}
/* trigger_port is locked and active */
ipc_port_make_send_locked(trigger_port);
/* now unlocked */
default_pager_triggers(default_pager,
hi_water, low_water,
HI_WAT_ALERT, trigger_port);
}
if (flags & LO_WAT_ALERT) {
trigger_port = trigger_name_to_port(trigger_name);
if(trigger_port == NULL) {
return EINVAL;
}
/* trigger_port is locked and active */
ipc_port_make_send_locked(trigger_port);
/* and now its unlocked */
default_pager_triggers(default_pager,
hi_water, low_water,
LO_WAT_ALERT, trigger_port);
}
/*
* Set thread scheduling priority and policy for the current thread
* it is assumed for the time being that the thread setting the alert
* is the same one which will be servicing it.
*
* XXX This does not belong in the kernel XXX
*/
{
thread_precedence_policy_data_t pre;
thread_extended_policy_data_t ext;
ext.timeshare = FALSE;
pre.importance = INT32_MAX;
thread_policy_set(current_thread(),
THREAD_EXTENDED_POLICY,
(thread_policy_t)&ext,
THREAD_EXTENDED_POLICY_COUNT);
thread_policy_set(current_thread(),
THREAD_PRECEDENCE_POLICY,
(thread_policy_t)&pre,
THREAD_PRECEDENCE_POLICY_COUNT);
}
current_thread()->options |= TH_OPT_VMPRIV;
return 0;
}
/*
* Routine: macx_swapon
* Function:
* Syscall interface to add a file to backing store
*/
int
macx_swapon(
char *filename,
int flags,
long size,
long priority)
{
struct vnode *vp = 0;
struct nameidata nd, *ndp;
struct proc *p = current_proc();
pager_file_t pf;
register int error;
kern_return_t kr;
mach_port_t backing_store;
memory_object_default_t default_pager;
int i;
boolean_t funnel_state;
struct vattr vattr;
AUDIT_MACH_SYSCALL_ENTER(AUE_SWAPON);
AUDIT_ARG(value, priority);
funnel_state = thread_funnel_set(kernel_flock, TRUE);
ndp = &nd;
if ((error = suser(p->p_ucred, &p->p_acflag)))
goto swapon_bailout;
if(default_pager_init_flag == 0) {
start_def_pager(NULL);
default_pager_init_flag = 1;
}
/*
* Get a vnode for the paging area.
*/
NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1, UIO_USERSPACE,
filename, p);
if ((error = namei(ndp)))
goto swapon_bailout;
vp = ndp->ni_vp;
if (vp->v_type != VREG) {
error = EINVAL;
VOP_UNLOCK(vp, 0, p);
goto swapon_bailout;
}
UBCINFOCHECK("macx_swapon", vp);
if (error = VOP_GETATTR(vp, &vattr, p->p_ucred, p)) {
VOP_UNLOCK(vp, 0, p);
goto swapon_bailout;
}
if (vattr.va_size < (u_quad_t)size) {
vattr_null(&vattr);
vattr.va_size = (u_quad_t)size;
error = VOP_SETATTR(vp, &vattr, p->p_ucred, p);
if (error) {
VOP_UNLOCK(vp, 0, p);
goto swapon_bailout;
}
}
/* add new backing store to list */
i = 0;
while(bs_port_table[i].vp != 0) {
if(i == MAX_BACKING_STORE)
break;
i++;
}
if(i == MAX_BACKING_STORE) {
error = ENOMEM;
VOP_UNLOCK(vp, 0, p);
goto swapon_bailout;
}
/* remember the vnode. This vnode has namei() reference */
bs_port_table[i].vp = vp;
/*
* Look to see if we are already paging to this file.
*/
/* make certain the copy send of kernel call will work */
default_pager = MEMORY_OBJECT_DEFAULT_NULL;
kr = host_default_memory_manager(host_priv_self(), &default_pager, 0);
if(kr != KERN_SUCCESS) {
error = EAGAIN;
VOP_UNLOCK(vp, 0, p);
bs_port_table[i].vp = 0;
goto swapon_bailout;
}
kr = default_pager_backing_store_create(default_pager,
-1, /* default priority */
0, /* default cluster size */
&backing_store);
memory_object_default_deallocate(default_pager);
if(kr != KERN_SUCCESS) {
error = ENOMEM;
VOP_UNLOCK(vp, 0, p);
bs_port_table[i].vp = 0;
goto swapon_bailout;
}
/*
* NOTE: we are able to supply PAGE_SIZE here instead of
* an actual record size or block number because:
* a: we do not support offsets from the beginning of the
* file (allowing for non page size/record modulo offsets.
* b: because allow paging will be done modulo page size
*/
VOP_UNLOCK(vp, 0, p);
kr = default_pager_add_file(backing_store, vp, PAGE_SIZE,
((int)vattr.va_size)/PAGE_SIZE);
if(kr != KERN_SUCCESS) {
bs_port_table[i].vp = 0;
if(kr == KERN_INVALID_ARGUMENT)
error = EINVAL;
else
error = ENOMEM;
goto swapon_bailout;
}
bs_port_table[i].bs = (void *)backing_store;
error = 0;
if (!ubc_hold(vp))
panic("macx_swapon: hold");
/* Mark this vnode as being used for swapfile */
SET(vp->v_flag, VSWAP);
ubc_setcred(vp, p);
/*
* take an extra reference on the vnode to keep
* vnreclaim() away from this vnode.
*/
VREF(vp);
/* Hold on to the namei reference to the paging file vnode */
vp = 0;
swapon_bailout:
if (vp) {
vrele(vp);
}
(void) thread_funnel_set(kernel_flock, FALSE);
AUDIT_MACH_SYSCALL_EXIT(error);
return(error);
}
/*
* Routine: mach_macx_triggers
* Function:
* Syscall interface to set the call backs for low and
* high water marks.
*/
int
mach_macx_triggers(
struct macx_triggers_args *args)
{
int hi_water = args->hi_water;
int low_water = args->low_water;
int flags = args->flags;
mach_port_t trigger_name = args->alert_port;
kern_return_t kr;
memory_object_default_t default_pager;
ipc_port_t trigger_port;
default_pager = MEMORY_OBJECT_DEFAULT_NULL;
kr = host_default_memory_manager(host_priv_self(),
&default_pager, 0);
if(kr != KERN_SUCCESS) {
return EINVAL;
}
if (((flags & SWAP_ENCRYPT_ON) && (flags & SWAP_ENCRYPT_OFF)) ||
((flags & SWAP_COMPACT_ENABLE) && (flags & SWAP_COMPACT_DISABLE))) {
/* can't have it both ways */
return EINVAL;
}
if (default_pager_init_flag == 0) {
start_def_pager(NULL);
default_pager_init_flag = 1;
}
if (flags & SWAP_ENCRYPT_ON) {
/* ENCRYPTED SWAP: tell default_pager to encrypt */
default_pager_triggers(default_pager,
0, 0,
SWAP_ENCRYPT_ON,
IP_NULL);
} else if (flags & SWAP_ENCRYPT_OFF) {
/* ENCRYPTED SWAP: tell default_pager not to encrypt */
default_pager_triggers(default_pager,
0, 0,
SWAP_ENCRYPT_OFF,
IP_NULL);
}
if (flags & USE_EMERGENCY_SWAP_FILE_FIRST) {
/*
* Time to switch to the emergency segment.
*/
return default_pager_triggers(default_pager,
0, 0,
USE_EMERGENCY_SWAP_FILE_FIRST,
IP_NULL);
}
if (flags & SWAP_FILE_CREATION_ERROR) {
/*
* For some reason, the dynamic pager failed to create a swap file.
*/
trigger_port = trigger_name_to_port(trigger_name);
if(trigger_port == NULL) {
return EINVAL;
}
/* trigger_port is locked and active */
ipc_port_make_send_locked(trigger_port);
/* now unlocked */
default_pager_triggers(default_pager,
0, 0,
SWAP_FILE_CREATION_ERROR,
trigger_port);
}
if (flags & HI_WAT_ALERT) {
trigger_port = trigger_name_to_port(trigger_name);
if(trigger_port == NULL) {
return EINVAL;
}
/* trigger_port is locked and active */
ipc_port_make_send_locked(trigger_port);
/* now unlocked */
default_pager_triggers(default_pager,
hi_water, low_water,
HI_WAT_ALERT, trigger_port);
}
if (flags & LO_WAT_ALERT) {
trigger_port = trigger_name_to_port(trigger_name);
if(trigger_port == NULL) {
return EINVAL;
}
/* trigger_port is locked and active */
ipc_port_make_send_locked(trigger_port);
/* and now its unlocked */
default_pager_triggers(default_pager,
hi_water, low_water,
LO_WAT_ALERT, trigger_port);
}
if (flags & PROC_RESUME) {
/*
* For this call, hi_water is used to pass in the pid of the process we want to resume
* or unthrottle. This is of course restricted to the superuser (checked inside of
* proc_resetpcontrol).
*/
return proc_resetpcontrol(hi_water);
}
/*
* Set thread scheduling priority and policy for the current thread
* it is assumed for the time being that the thread setting the alert
* is the same one which will be servicing it.
*
* XXX This does not belong in the kernel XXX
*/
if (flags & HI_WAT_ALERT) {
thread_precedence_policy_data_t pre;
thread_extended_policy_data_t ext;
ext.timeshare = FALSE;
pre.importance = INT32_MAX;
thread_policy_set(current_thread(),
THREAD_EXTENDED_POLICY,
(thread_policy_t)&ext,
THREAD_EXTENDED_POLICY_COUNT);
thread_policy_set(current_thread(),
THREAD_PRECEDENCE_POLICY,
(thread_policy_t)&pre,
THREAD_PRECEDENCE_POLICY_COUNT);
current_thread()->options |= TH_OPT_VMPRIV;
}
if (flags & (SWAP_COMPACT_DISABLE | SWAP_COMPACT_ENABLE)) {
return macx_backing_store_compaction(flags & (SWAP_COMPACT_DISABLE | SWAP_COMPACT_ENABLE));
}
return 0;
}
void
default_pager_initialize(
mach_port_t host_port)
{
kern_return_t kr;
static char here[] = "default_pager_initialize";
/*
* Initial thread and task ports.
*/
default_pager_self = mach_task_self();
default_pager_default_thread = mach_thread_self();
PRINTF_LOCK_INIT();
/*
* Make ourselves unswappable.
*/
kr = task_swappable(default_pager_host_port, default_pager_self, FALSE);
if (kr != KERN_SUCCESS)
dprintf(("task_swappable failed 0x%x %s\n",
kr, mach_error_string(kr)));
/*
* Exported DMM port.
*/
kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE,
&default_pager_default_port);
if (kr != KERN_SUCCESS)
Panic("default port");
/*
* Port sets.
*/
kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET,
&default_pager_internal_set);
if (kr != KERN_SUCCESS)
Panic("internal set");
kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET,
&default_pager_external_set);
if (kr != KERN_SUCCESS)
Panic("external set");
/*
* Export pager interfaces.
*/
#ifdef USER_PAGER
if ((kr = netname_check_in(name_server_port, "UserPager",
default_pager_self,
default_pager_default_port))
!= KERN_SUCCESS) {
dprintf(("netname_check_in returned 0x%x %s\n",
kr, mach_error_string(kr)));
exit(1);
}
#else /* USER_PAGER */
{
int clsize;
memory_object_t DMM;
/* get a send right for vm_set_default_memory_manager */
kr = mach_port_insert_right(default_pager_self,
default_pager_default_port,
default_pager_default_port,
MACH_MSG_TYPE_MAKE_SEND);
DMM = default_pager_default_port;
clsize = (vm_page_size << vstruct_def_clshift);
kr = host_default_memory_manager(host_port, &DMM, clsize);
if ((kr != KERN_SUCCESS) || (DMM != MACH_PORT_NULL))
Panic("default memory manager");
/* release the extra send right */
(void) mach_port_mod_refs(default_pager_self,
default_pager_default_port,
MACH_PORT_RIGHT_SEND,
-1);
}
#endif /* USER_PAGER */
kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET,
&default_pager_default_set);
if (kr != KERN_SUCCESS)
Panic("default set");
kr = mach_port_move_member(default_pager_self,
default_pager_default_port,
default_pager_default_set);
if (kr != KERN_SUCCESS)
Panic("set up default");
/*
* Arrange for wiring privileges.
*/
wire_setup(host_port);
/*
* Find out how many CPUs we have, to determine the number
* of threads to create.
*/
if (default_pager_internal_count == 0) {
host_basic_info_data_t h_info;
mach_msg_type_number_t h_info_count;
h_info_count = HOST_BASIC_INFO_COUNT;
(void) host_info(host_port, HOST_BASIC_INFO,
(host_info_t) &h_info, &h_info_count);
/*
* Random computation to get more parallelism on
* multiprocessors.
*/
default_pager_internal_count = ((h_info.avail_cpus > 32)
? 32
: h_info.avail_cpus) / 4 + 3;
}
/*
* Vm variables.
*/
vm_page_mask = vm_page_size - 1;
vm_page_shift = log2(vm_page_size);
/*
* List of all vstructs.
*/
VSL_LOCK_INIT();
queue_init(&vstruct_list.vsl_queue);
queue_init(&vstruct_list.vsl_leak_queue);
vstruct_list.vsl_count = 0;
VSTATS_LOCK_INIT(&global_stats.gs_lock);
bs_initialize();
}