/* * Routine: macx_swapon * Function: * Syscall interface to add a file to backing store */ int macx_swapon( struct macx_swapon_args *args) { int size = args->size; vnode_t vp = (vnode_t)NULL; struct nameidata nd, *ndp; register int error; kern_return_t kr; mach_port_t backing_store; memory_object_default_t default_pager; int i; boolean_t funnel_state; off_t file_size; vfs_context_t ctx = vfs_context_current(); struct proc *p = current_proc(); int dp_cluster_size; AUDIT_MACH_SYSCALL_ENTER(AUE_SWAPON); AUDIT_ARG(value32, args->priority); funnel_state = thread_funnel_set(kernel_flock, TRUE); ndp = &nd; if ((error = suser(kauth_cred_get(), 0))) goto swapon_bailout; /* * Get a vnode for the paging area. */ NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1, ((IS_64BIT_PROCESS(p)) ? UIO_USERSPACE64 : UIO_USERSPACE32), (user_addr_t) args->filename, ctx); if ((error = namei(ndp))) goto swapon_bailout; nameidone(ndp); vp = ndp->ni_vp; if (vp->v_type != VREG) { error = EINVAL; goto swapon_bailout; } /* get file size */ if ((error = vnode_size(vp, &file_size, ctx)) != 0) goto swapon_bailout; #if CONFIG_MACF vnode_lock(vp); error = mac_system_check_swapon(vfs_context_ucred(ctx), vp); vnode_unlock(vp); if (error) goto swapon_bailout; #endif /* resize to desired size if it's too small */ if ((file_size < (off_t)size) && ((error = vnode_setsize(vp, (off_t)size, 0, ctx)) != 0)) goto swapon_bailout; if (default_pager_init_flag == 0) { start_def_pager(NULL); default_pager_init_flag = 1; } /* add new backing store to list */ i = 0; while(bs_port_table[i].vp != 0) { if(i == MAX_BACKING_STORE) break; i++; } if(i == MAX_BACKING_STORE) { error = ENOMEM; goto swapon_bailout; } /* remember the vnode. This vnode has namei() reference */ bs_port_table[i].vp = vp; /* * Look to see if we are already paging to this file. */ /* make certain the copy send of kernel call will work */ default_pager = MEMORY_OBJECT_DEFAULT_NULL; kr = host_default_memory_manager(host_priv_self(), &default_pager, 0); if(kr != KERN_SUCCESS) { error = EAGAIN; bs_port_table[i].vp = 0; goto swapon_bailout; } if (vp->v_mount->mnt_kern_flag & MNTK_SSD) { /* * keep the cluster size small since the * seek cost is effectively 0 which means * we don't care much about fragmentation */ dp_isssd = TRUE; dp_cluster_size = 2 * PAGE_SIZE; } else { /* * use the default cluster size */ dp_isssd = FALSE; dp_cluster_size = 0; } kr = default_pager_backing_store_create(default_pager, -1, /* default priority */ dp_cluster_size, &backing_store); memory_object_default_deallocate(default_pager); if(kr != KERN_SUCCESS) { error = ENOMEM; bs_port_table[i].vp = 0; goto swapon_bailout; } /* Mark this vnode as being used for swapfile */ vnode_lock_spin(vp); SET(vp->v_flag, VSWAP); vnode_unlock(vp); /* * NOTE: we are able to supply PAGE_SIZE here instead of * an actual record size or block number because: * a: we do not support offsets from the beginning of the * file (allowing for non page size/record modulo offsets. * b: because allow paging will be done modulo page size */ kr = default_pager_add_file(backing_store, (vnode_ptr_t) vp, PAGE_SIZE, (int)(file_size/PAGE_SIZE)); if(kr != KERN_SUCCESS) { bs_port_table[i].vp = 0; if(kr == KERN_INVALID_ARGUMENT) error = EINVAL; else error = ENOMEM; /* This vnode is not to be used for swapfile */ vnode_lock_spin(vp); CLR(vp->v_flag, VSWAP); vnode_unlock(vp); goto swapon_bailout; } bs_port_table[i].bs = (void *)backing_store; error = 0; ubc_setthreadcred(vp, p, current_thread()); /* * take a long term reference on the vnode to keep * vnreclaim() away from this vnode. */ vnode_ref(vp); swapon_bailout: if (vp) { vnode_put(vp); } (void) thread_funnel_set(kernel_flock, FALSE); AUDIT_MACH_SYSCALL_EXIT(error); return(error); }
/* * Routine: macx_swapinfo * Function: * Syscall interface to get general swap statistics */ int macx_swapinfo( memory_object_size_t *total_p, memory_object_size_t *avail_p, vm_size_t *pagesize_p, boolean_t *encrypted_p) { int error; memory_object_default_t default_pager; default_pager_info_64_t dpi64; kern_return_t kr; error = 0; /* * Get a handle on the default pager. */ default_pager = MEMORY_OBJECT_DEFAULT_NULL; kr = host_default_memory_manager(host_priv_self(), &default_pager, 0); if (kr != KERN_SUCCESS) { error = EAGAIN; /* XXX why EAGAIN ? */ goto done; } if (default_pager == MEMORY_OBJECT_DEFAULT_NULL) { /* * The default pager has not initialized yet, * so it can't be using any swap space at all. */ *total_p = 0; *avail_p = 0; *pagesize_p = 0; *encrypted_p = FALSE; goto done; } /* * Get swap usage data from default pager. */ kr = default_pager_info_64(default_pager, &dpi64); if (kr != KERN_SUCCESS) { error = ENOTSUP; goto done; } /* * Provide default pager info to caller. */ *total_p = dpi64.dpi_total_space; *avail_p = dpi64.dpi_free_space; *pagesize_p = dpi64.dpi_page_size; if (dpi64.dpi_flags & DPI_ENCRYPTED) { *encrypted_p = TRUE; } else { *encrypted_p = FALSE; } done: if (default_pager != MEMORY_OBJECT_DEFAULT_NULL) { /* release our handle on default pager */ memory_object_default_deallocate(default_pager); } return error; }
void default_pager_initialize(void) { kern_return_t kr; __unused static char here[] = "default_pager_initialize"; lck_grp_attr_setdefault(&default_pager_lck_grp_attr); lck_grp_init(&default_pager_lck_grp, "default_pager", &default_pager_lck_grp_attr); lck_attr_setdefault(&default_pager_lck_attr); /* * Vm variables. */ #ifndef MACH_KERNEL vm_page_mask = vm_page_size - 1; assert((unsigned int) vm_page_size == vm_page_size); vm_page_shift = local_log2((unsigned int) vm_page_size); #endif /* * List of all vstructs. */ vstruct_zone = zinit(sizeof(struct vstruct), 10000 * sizeof(struct vstruct), 8192, "vstruct zone"); zone_change(vstruct_zone, Z_CALLERACCT, FALSE); zone_change(vstruct_zone, Z_NOENCRYPT, TRUE); VSL_LOCK_INIT(); queue_init(&vstruct_list.vsl_queue); vstruct_list.vsl_count = 0; VSTATS_LOCK_INIT(&global_stats.gs_lock); bs_initialize(); /* * Exported DMM port. */ default_pager_object = ipc_port_alloc_kernel(); /* * Export pager interfaces. */ #ifdef USER_PAGER if ((kr = netname_check_in(name_server_port, "UserPager", default_pager_self, default_pager_object)) != KERN_SUCCESS) { dprintf(("netname_check_in returned 0x%x\n", kr)); exit(1); } #else /* USER_PAGER */ { unsigned int clsize; memory_object_default_t dmm; dmm = default_pager_object; assert((unsigned int) vm_page_size == vm_page_size); clsize = ((unsigned int) vm_page_size << vstruct_def_clshift); kr = host_default_memory_manager(host_priv_self(), &dmm, clsize); if ((kr != KERN_SUCCESS) || (dmm != MEMORY_OBJECT_DEFAULT_NULL)) Panic("default memory manager"); } #endif /* USER_PAGER */ }
/* * Routine: macx_triggers * Function: * Syscall interface to set the call backs for low and * high water marks. */ int macx_triggers( struct macx_triggers_args *args) { int hi_water = args->hi_water; int low_water = args->low_water; int flags = args->flags; mach_port_t trigger_name = args->alert_port; kern_return_t kr; memory_object_default_t default_pager; ipc_port_t trigger_port; default_pager = MEMORY_OBJECT_DEFAULT_NULL; kr = host_default_memory_manager(host_priv_self(), &default_pager, 0); if(kr != KERN_SUCCESS) { return EINVAL; } if ((flags & SWAP_ENCRYPT_ON) && (flags & SWAP_ENCRYPT_OFF)) { /* can't have it both ways */ return EINVAL; } if (default_pager_init_flag == 0) { start_def_pager(NULL); default_pager_init_flag = 1; } if (flags & SWAP_ENCRYPT_ON) { /* ENCRYPTED SWAP: tell default_pager to encrypt */ default_pager_triggers(default_pager, 0, 0, SWAP_ENCRYPT_ON, IP_NULL); } else if (flags & SWAP_ENCRYPT_OFF) { /* ENCRYPTED SWAP: tell default_pager not to encrypt */ default_pager_triggers(default_pager, 0, 0, SWAP_ENCRYPT_OFF, IP_NULL); } if (flags & HI_WAT_ALERT) { trigger_port = trigger_name_to_port(trigger_name); if(trigger_port == NULL) { return EINVAL; } /* trigger_port is locked and active */ ipc_port_make_send_locked(trigger_port); /* now unlocked */ default_pager_triggers(default_pager, hi_water, low_water, HI_WAT_ALERT, trigger_port); } if (flags & LO_WAT_ALERT) { trigger_port = trigger_name_to_port(trigger_name); if(trigger_port == NULL) { return EINVAL; } /* trigger_port is locked and active */ ipc_port_make_send_locked(trigger_port); /* and now its unlocked */ default_pager_triggers(default_pager, hi_water, low_water, LO_WAT_ALERT, trigger_port); } /* * Set thread scheduling priority and policy for the current thread * it is assumed for the time being that the thread setting the alert * is the same one which will be servicing it. * * XXX This does not belong in the kernel XXX */ { thread_precedence_policy_data_t pre; thread_extended_policy_data_t ext; ext.timeshare = FALSE; pre.importance = INT32_MAX; thread_policy_set(current_thread(), THREAD_EXTENDED_POLICY, (thread_policy_t)&ext, THREAD_EXTENDED_POLICY_COUNT); thread_policy_set(current_thread(), THREAD_PRECEDENCE_POLICY, (thread_policy_t)&pre, THREAD_PRECEDENCE_POLICY_COUNT); } current_thread()->options |= TH_OPT_VMPRIV; return 0; }
/* * Routine: macx_swapon * Function: * Syscall interface to add a file to backing store */ int macx_swapon( char *filename, int flags, long size, long priority) { struct vnode *vp = 0; struct nameidata nd, *ndp; struct proc *p = current_proc(); pager_file_t pf; register int error; kern_return_t kr; mach_port_t backing_store; memory_object_default_t default_pager; int i; boolean_t funnel_state; struct vattr vattr; AUDIT_MACH_SYSCALL_ENTER(AUE_SWAPON); AUDIT_ARG(value, priority); funnel_state = thread_funnel_set(kernel_flock, TRUE); ndp = &nd; if ((error = suser(p->p_ucred, &p->p_acflag))) goto swapon_bailout; if(default_pager_init_flag == 0) { start_def_pager(NULL); default_pager_init_flag = 1; } /* * Get a vnode for the paging area. */ NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1, UIO_USERSPACE, filename, p); if ((error = namei(ndp))) goto swapon_bailout; vp = ndp->ni_vp; if (vp->v_type != VREG) { error = EINVAL; VOP_UNLOCK(vp, 0, p); goto swapon_bailout; } UBCINFOCHECK("macx_swapon", vp); if (error = VOP_GETATTR(vp, &vattr, p->p_ucred, p)) { VOP_UNLOCK(vp, 0, p); goto swapon_bailout; } if (vattr.va_size < (u_quad_t)size) { vattr_null(&vattr); vattr.va_size = (u_quad_t)size; error = VOP_SETATTR(vp, &vattr, p->p_ucred, p); if (error) { VOP_UNLOCK(vp, 0, p); goto swapon_bailout; } } /* add new backing store to list */ i = 0; while(bs_port_table[i].vp != 0) { if(i == MAX_BACKING_STORE) break; i++; } if(i == MAX_BACKING_STORE) { error = ENOMEM; VOP_UNLOCK(vp, 0, p); goto swapon_bailout; } /* remember the vnode. This vnode has namei() reference */ bs_port_table[i].vp = vp; /* * Look to see if we are already paging to this file. */ /* make certain the copy send of kernel call will work */ default_pager = MEMORY_OBJECT_DEFAULT_NULL; kr = host_default_memory_manager(host_priv_self(), &default_pager, 0); if(kr != KERN_SUCCESS) { error = EAGAIN; VOP_UNLOCK(vp, 0, p); bs_port_table[i].vp = 0; goto swapon_bailout; } kr = default_pager_backing_store_create(default_pager, -1, /* default priority */ 0, /* default cluster size */ &backing_store); memory_object_default_deallocate(default_pager); if(kr != KERN_SUCCESS) { error = ENOMEM; VOP_UNLOCK(vp, 0, p); bs_port_table[i].vp = 0; goto swapon_bailout; } /* * NOTE: we are able to supply PAGE_SIZE here instead of * an actual record size or block number because: * a: we do not support offsets from the beginning of the * file (allowing for non page size/record modulo offsets. * b: because allow paging will be done modulo page size */ VOP_UNLOCK(vp, 0, p); kr = default_pager_add_file(backing_store, vp, PAGE_SIZE, ((int)vattr.va_size)/PAGE_SIZE); if(kr != KERN_SUCCESS) { bs_port_table[i].vp = 0; if(kr == KERN_INVALID_ARGUMENT) error = EINVAL; else error = ENOMEM; goto swapon_bailout; } bs_port_table[i].bs = (void *)backing_store; error = 0; if (!ubc_hold(vp)) panic("macx_swapon: hold"); /* Mark this vnode as being used for swapfile */ SET(vp->v_flag, VSWAP); ubc_setcred(vp, p); /* * take an extra reference on the vnode to keep * vnreclaim() away from this vnode. */ VREF(vp); /* Hold on to the namei reference to the paging file vnode */ vp = 0; swapon_bailout: if (vp) { vrele(vp); } (void) thread_funnel_set(kernel_flock, FALSE); AUDIT_MACH_SYSCALL_EXIT(error); return(error); }
/* * Routine: mach_macx_triggers * Function: * Syscall interface to set the call backs for low and * high water marks. */ int mach_macx_triggers( struct macx_triggers_args *args) { int hi_water = args->hi_water; int low_water = args->low_water; int flags = args->flags; mach_port_t trigger_name = args->alert_port; kern_return_t kr; memory_object_default_t default_pager; ipc_port_t trigger_port; default_pager = MEMORY_OBJECT_DEFAULT_NULL; kr = host_default_memory_manager(host_priv_self(), &default_pager, 0); if(kr != KERN_SUCCESS) { return EINVAL; } if (((flags & SWAP_ENCRYPT_ON) && (flags & SWAP_ENCRYPT_OFF)) || ((flags & SWAP_COMPACT_ENABLE) && (flags & SWAP_COMPACT_DISABLE))) { /* can't have it both ways */ return EINVAL; } if (default_pager_init_flag == 0) { start_def_pager(NULL); default_pager_init_flag = 1; } if (flags & SWAP_ENCRYPT_ON) { /* ENCRYPTED SWAP: tell default_pager to encrypt */ default_pager_triggers(default_pager, 0, 0, SWAP_ENCRYPT_ON, IP_NULL); } else if (flags & SWAP_ENCRYPT_OFF) { /* ENCRYPTED SWAP: tell default_pager not to encrypt */ default_pager_triggers(default_pager, 0, 0, SWAP_ENCRYPT_OFF, IP_NULL); } if (flags & USE_EMERGENCY_SWAP_FILE_FIRST) { /* * Time to switch to the emergency segment. */ return default_pager_triggers(default_pager, 0, 0, USE_EMERGENCY_SWAP_FILE_FIRST, IP_NULL); } if (flags & SWAP_FILE_CREATION_ERROR) { /* * For some reason, the dynamic pager failed to create a swap file. */ trigger_port = trigger_name_to_port(trigger_name); if(trigger_port == NULL) { return EINVAL; } /* trigger_port is locked and active */ ipc_port_make_send_locked(trigger_port); /* now unlocked */ default_pager_triggers(default_pager, 0, 0, SWAP_FILE_CREATION_ERROR, trigger_port); } if (flags & HI_WAT_ALERT) { trigger_port = trigger_name_to_port(trigger_name); if(trigger_port == NULL) { return EINVAL; } /* trigger_port is locked and active */ ipc_port_make_send_locked(trigger_port); /* now unlocked */ default_pager_triggers(default_pager, hi_water, low_water, HI_WAT_ALERT, trigger_port); } if (flags & LO_WAT_ALERT) { trigger_port = trigger_name_to_port(trigger_name); if(trigger_port == NULL) { return EINVAL; } /* trigger_port is locked and active */ ipc_port_make_send_locked(trigger_port); /* and now its unlocked */ default_pager_triggers(default_pager, hi_water, low_water, LO_WAT_ALERT, trigger_port); } if (flags & PROC_RESUME) { /* * For this call, hi_water is used to pass in the pid of the process we want to resume * or unthrottle. This is of course restricted to the superuser (checked inside of * proc_resetpcontrol). */ return proc_resetpcontrol(hi_water); } /* * Set thread scheduling priority and policy for the current thread * it is assumed for the time being that the thread setting the alert * is the same one which will be servicing it. * * XXX This does not belong in the kernel XXX */ if (flags & HI_WAT_ALERT) { thread_precedence_policy_data_t pre; thread_extended_policy_data_t ext; ext.timeshare = FALSE; pre.importance = INT32_MAX; thread_policy_set(current_thread(), THREAD_EXTENDED_POLICY, (thread_policy_t)&ext, THREAD_EXTENDED_POLICY_COUNT); thread_policy_set(current_thread(), THREAD_PRECEDENCE_POLICY, (thread_policy_t)&pre, THREAD_PRECEDENCE_POLICY_COUNT); current_thread()->options |= TH_OPT_VMPRIV; } if (flags & (SWAP_COMPACT_DISABLE | SWAP_COMPACT_ENABLE)) { return macx_backing_store_compaction(flags & (SWAP_COMPACT_DISABLE | SWAP_COMPACT_ENABLE)); } return 0; }
void default_pager_initialize( mach_port_t host_port) { kern_return_t kr; static char here[] = "default_pager_initialize"; /* * Initial thread and task ports. */ default_pager_self = mach_task_self(); default_pager_default_thread = mach_thread_self(); PRINTF_LOCK_INIT(); /* * Make ourselves unswappable. */ kr = task_swappable(default_pager_host_port, default_pager_self, FALSE); if (kr != KERN_SUCCESS) dprintf(("task_swappable failed 0x%x %s\n", kr, mach_error_string(kr))); /* * Exported DMM port. */ kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE, &default_pager_default_port); if (kr != KERN_SUCCESS) Panic("default port"); /* * Port sets. */ kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET, &default_pager_internal_set); if (kr != KERN_SUCCESS) Panic("internal set"); kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET, &default_pager_external_set); if (kr != KERN_SUCCESS) Panic("external set"); /* * Export pager interfaces. */ #ifdef USER_PAGER if ((kr = netname_check_in(name_server_port, "UserPager", default_pager_self, default_pager_default_port)) != KERN_SUCCESS) { dprintf(("netname_check_in returned 0x%x %s\n", kr, mach_error_string(kr))); exit(1); } #else /* USER_PAGER */ { int clsize; memory_object_t DMM; /* get a send right for vm_set_default_memory_manager */ kr = mach_port_insert_right(default_pager_self, default_pager_default_port, default_pager_default_port, MACH_MSG_TYPE_MAKE_SEND); DMM = default_pager_default_port; clsize = (vm_page_size << vstruct_def_clshift); kr = host_default_memory_manager(host_port, &DMM, clsize); if ((kr != KERN_SUCCESS) || (DMM != MACH_PORT_NULL)) Panic("default memory manager"); /* release the extra send right */ (void) mach_port_mod_refs(default_pager_self, default_pager_default_port, MACH_PORT_RIGHT_SEND, -1); } #endif /* USER_PAGER */ kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET, &default_pager_default_set); if (kr != KERN_SUCCESS) Panic("default set"); kr = mach_port_move_member(default_pager_self, default_pager_default_port, default_pager_default_set); if (kr != KERN_SUCCESS) Panic("set up default"); /* * Arrange for wiring privileges. */ wire_setup(host_port); /* * Find out how many CPUs we have, to determine the number * of threads to create. */ if (default_pager_internal_count == 0) { host_basic_info_data_t h_info; mach_msg_type_number_t h_info_count; h_info_count = HOST_BASIC_INFO_COUNT; (void) host_info(host_port, HOST_BASIC_INFO, (host_info_t) &h_info, &h_info_count); /* * Random computation to get more parallelism on * multiprocessors. */ default_pager_internal_count = ((h_info.avail_cpus > 32) ? 32 : h_info.avail_cpus) / 4 + 3; } /* * Vm variables. */ vm_page_mask = vm_page_size - 1; vm_page_shift = log2(vm_page_size); /* * List of all vstructs. */ VSL_LOCK_INIT(); queue_init(&vstruct_list.vsl_queue); queue_init(&vstruct_list.vsl_leak_queue); vstruct_list.vsl_count = 0; VSTATS_LOCK_INIT(&global_stats.gs_lock); bs_initialize(); }