Object* agmtlist_get_next_agreement_for_replica (Replica *r, Object *prev)
{
const Slapi_DN *replica_root;
Slapi_DN *agmt_root;
Object *obj;
Repl_Agmt *agmt;
if (r == NULL)
{
/* ONREPL - log error */
return NULL;
}
replica_root = replica_get_root(r);
if (prev)
obj = objset_next_obj(agmt_set, prev);
else
obj = objset_first_obj(agmt_set);
while (obj)
{
agmt = (Repl_Agmt*)object_get_data (obj);
PR_ASSERT (agmt);
agmt_root = agmt_get_replarea(agmt);
PR_ASSERT (agmt_root);
if (slapi_sdn_compare (replica_root, agmt_root) == 0)
{
slapi_sdn_free (&agmt_root);
return obj;
}
slapi_sdn_free (&agmt_root);
obj = objset_next_obj(agmt_set, obj);
}
return NULL;
}
void
allinstance_set_not_busy(struct ldbminfo *li)
{
ldbm_instance *inst;
Object *inst_obj;
/* server is up -- mark all backends busy */
for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
inst = (ldbm_instance *)object_get_data(inst_obj);
instance_set_not_busy(inst);
}
}
/* Walks down the set of instances, stopping each one. */
int ldbm_instance_stopall(struct ldbminfo *li)
{
Object *inst_obj;
ldbm_instance *inst;
inst_obj = objset_first_obj(li->li_instance_set);
while (inst_obj != NULL) {
inst = (ldbm_instance *) object_get_data(inst_obj);
ldbm_instance_stop(inst->inst_be);
inst_obj = objset_next_obj(li->li_instance_set, inst_obj);
}
return 0;
}
void
allinstance_set_busy(struct ldbminfo *li)
{
ldbm_instance *inst;
Object *inst_obj;
/* server is up -- mark all backends busy */
for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
inst = (ldbm_instance *)object_get_data(inst_obj);
if (instance_set_busy(inst)) {
LDAPDebug1Arg(LDAP_DEBUG_TRACE, "could not set instance [%s] as busy, probably already busy\n",
inst->inst_name);
}
}
}
/*
* Find the replication agreement whose entry DN matches the given DN.
* Object is returned referenced, so be sure to release it when
* finished.
*/
Repl_Agmt *
agmtlist_get_by_agmt_name(const Slapi_DN *agmt_name)
{
Repl_Agmt *ra = NULL;
Object *ro;
for (ro = objset_first_obj(agmt_set); NULL != ro;
ro = objset_next_obj(agmt_set, ro))
{
ra = (Repl_Agmt *)object_get_data(ro);
if (agmt_matches_name(ra, agmt_name))
{
break;
}
}
return ra;
}
/*
* Notify each replication agreement about an update.
*/
void
agmtlist_notify_all(Slapi_PBlock *pb)
{
Repl_Agmt *ra;
Object *ro;
if (NULL != agmt_set)
{
ro = objset_first_obj(agmt_set);
while (NULL != ro)
{
ra = (Repl_Agmt *)object_get_data(ro);
agmt_notify_change(ra, pb);
ro = objset_next_obj(agmt_set, ro);
}
}
}
/* Currently this function doesn't bump
* the ref count of the instance returned.
*/
ldbm_instance *
ldbm_instance_find_by_name(struct ldbminfo *li, char *name)
{
Object *inst_obj;
ldbm_instance *inst;
inst_obj = objset_first_obj(li->li_instance_set);
while (inst_obj != NULL) {
inst = (ldbm_instance *) object_get_data(inst_obj);
if (!strcasecmp(inst->inst_name, name)) {
/* Currently we release the object here. There is no
* function for callers of this function to call to
* release the object.
*/
object_release(inst_obj);
return inst;
}
inst_obj = objset_next_obj(li->li_instance_set, inst_obj);
}
return NULL;
}
int
is_anyinstance_busy(struct ldbminfo *li)
{
ldbm_instance *inst;
Object *inst_obj;
int rval = 0;
/* server is up -- mark all backends busy */
for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
inst = (ldbm_instance *)object_get_data(inst_obj);
PR_Lock(inst->inst_config_mutex);
rval = inst->inst_flags & INST_FLAG_BUSY;
PR_Unlock(inst->inst_config_mutex);
if (0 != rval) {
break;
}
}
if (inst_obj)
object_release(inst_obj);
return rval;
}
void
agmtlist_shutdown()
{
Repl_Agmt *ra;
Object *ro;
Object *next_ro;
ro = objset_first_obj(agmt_set);
while (NULL != ro)
{
ra = (Repl_Agmt *)object_get_data(ro);
agmt_stop(ra);
agmt_update_consumer_ruv (ra);
next_ro = objset_next_obj(agmt_set, ro);
/* Object ro was released in objset_next_obj,
* but the address ro can be still used to remove ro from objset. */
objset_remove_obj(agmt_set, ro);
ro = next_ro;
}
objset_delete(&agmt_set);
agmt_set = NULL;
}
/* Walks down the set of instances, starting each one. */
int
ldbm_instance_startall(struct ldbminfo *li)
{
Object *inst_obj;
ldbm_instance *inst;
int rc = 0;
inst_obj = objset_first_obj(li->li_instance_set);
while (inst_obj != NULL) {
int rc1;
inst = (ldbm_instance *) object_get_data(inst_obj);
ldbm_instance_set_flags(inst);
rc1 = ldbm_instance_start(inst->inst_be);
if (rc1 != 0) {
rc = rc1;
} else {
vlv_init(inst);
slapi_mtn_be_started(inst->inst_be);
}
inst_obj = objset_next_obj(li->li_instance_set, inst_obj);
}
return rc;
}
/*
* Start the LDBM plugin, and all its instances.
*/
int
ldbm_back_start( Slapi_PBlock *pb )
{
struct ldbminfo *li;
char *home_dir;
int action;
int retval;
int issane = 0;
PRUint64 total_cache_size = 0;
size_t pagesize = 0;
size_t pages = 0;
size_t procpages = 0;
size_t availpages = 0;
char *msg = ""; /* This will be set by one of the two cache sizing paths below. */
char s[32]; /* big enough to hold %ld */
unsigned long cache_size_to_configure = 0;
int zone_pages;
int db_pages;
int entry_pages;
int import_pages;
size_t zone_size;
size_t import_size;
size_t total_size;
Object *inst_obj;
ldbm_instance *inst;
PRUint64 cache_size;
PRUint64 dncache_size;
PRUint64 db_size;
#ifndef LINUX
PRUint64 memsize = pages * pagesize;
#endif
slapi_log_err(SLAPI_LOG_TRACE, "ldbm_back_start", "ldbm backend starting\n");
slapi_pblock_get( pb, SLAPI_PLUGIN_PRIVATE, &li );
/* parse the config file here */
if (0 != ldbm_config_load_dse_info(li)) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Loading database configuration failed\n");
return SLAPI_FAIL_GENERAL;
}
/* register with the binder-based resource limit subsystem so that */
/* lookthroughlimit can be supported on a per-connection basis. */
if ( slapi_reslimit_register( SLAPI_RESLIMIT_TYPE_INT,
LDBM_LOOKTHROUGHLIMIT_AT, &li->li_reslimit_lookthrough_handle )
!= SLAPI_RESLIMIT_STATUS_SUCCESS ) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Resource limit registration failed for lookthroughlimit\n");
return SLAPI_FAIL_GENERAL;
}
/* register with the binder-based resource limit subsystem so that */
/* allidslimit (aka idlistscanlimit) can be supported on a per-connection basis. */
if ( slapi_reslimit_register( SLAPI_RESLIMIT_TYPE_INT,
LDBM_ALLIDSLIMIT_AT, &li->li_reslimit_allids_handle )
!= SLAPI_RESLIMIT_STATUS_SUCCESS ) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Resource limit registration failed for allidslimit\n");
return SLAPI_FAIL_GENERAL;
}
/* register with the binder-based resource limit subsystem so that */
/* pagedlookthroughlimit can be supported on a per-connection basis. */
if ( slapi_reslimit_register( SLAPI_RESLIMIT_TYPE_INT,
LDBM_PAGEDLOOKTHROUGHLIMIT_AT, &li->li_reslimit_pagedlookthrough_handle )
!= SLAPI_RESLIMIT_STATUS_SUCCESS ) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Resource limit registration failed for pagedlookthroughlimit\n");
return SLAPI_FAIL_GENERAL;
}
/* register with the binder-based resource limit subsystem so that */
/* pagedallidslimit (aka idlistscanlimit) can be supported on a per-connection basis. */
if ( slapi_reslimit_register( SLAPI_RESLIMIT_TYPE_INT,
LDBM_PAGEDALLIDSLIMIT_AT, &li->li_reslimit_pagedallids_handle )
!= SLAPI_RESLIMIT_STATUS_SUCCESS ) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Resource limit registration failed for pagedallidslimit\n");
return SLAPI_FAIL_GENERAL;
}
/* lookthrough limit for the rangesearch */
if ( slapi_reslimit_register( SLAPI_RESLIMIT_TYPE_INT,
LDBM_RANGELOOKTHROUGHLIMIT_AT, &li->li_reslimit_rangelookthrough_handle )
!= SLAPI_RESLIMIT_STATUS_SUCCESS ) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Resource limit registration failed for rangelookthroughlimit\n");
return SLAPI_FAIL_GENERAL;
}
/* If the db directory hasn't been set yet, we need to set it to
* the default. */
if (NULL == li->li_directory || '\0' == li->li_directory[0]) {
/* "get default" is a special string that tells the config
* routines to figure out the default db directory by
* reading cn=config. */
ldbm_config_internal_set(li, CONFIG_DIRECTORY, "get default");
}
/* sanity check the autosizing values,
no value or sum of values larger than 100.
*/
if ((li->li_cache_autosize > 100) ||
(li->li_cache_autosize_split > 100) ||
(li->li_import_cache_autosize > 100) ||
((li->li_cache_autosize > 0) && (li->li_import_cache_autosize > 0) &&
(li->li_cache_autosize + li->li_import_cache_autosize > 100))) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Cache autosizing: bad settings, "
"value or sum of values can not larger than 100.\n");
} else {
if (util_info_sys_pages(&pagesize, &pages, &procpages, &availpages) != 0) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Unable to determine system page limits\n");
return SLAPI_FAIL_GENERAL;
}
if (pagesize) {
if (li->li_cache_autosize == 0) {
/* First, set our message. */
msg = "This can be corrected by altering the values of nsslapd-dbcachesize, nsslapd-cachememsize and nsslapd-dncachememsize\n";
for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
inst = (ldbm_instance *)object_get_data(inst_obj);
cache_size = (PRUint64)cache_get_max_size(&(inst->inst_cache));
db_size = dblayer_get_id2entry_size(inst);
if (cache_size < db_size) {
slapi_log_err(SLAPI_LOG_NOTICE,
"ldbm_back_start - "
"%s: entry cache size %llu B is "
"less than db size %llu B; "
"We recommend to increase the entry cache size "
"nsslapd-cachememsize.\n",
inst->inst_name, cache_size, db_size);
} else {
slapi_log_err(SLAPI_LOG_BACKLDBM,
"ldbm_back_start", "%s: entry cache size: %lu B; db size: %lu B\n",
inst->inst_name, cache_size, db_size);
}
/* Get the dn_cachesize */
dncache_size = (PRUint64)cache_get_max_size(&(inst->inst_dncache));
total_cache_size += cache_size + dncache_size;
slapi_log_err(SLAPI_LOG_BACKLDBM,
"ldbm_back_start", "total cache size: %lu B; \n",
total_cache_size);
}
slapi_log_err(SLAPI_LOG_BACKLDBM,
"ldbm_back_start", "Total entry cache size: %lu B; "
"dbcache size: %lu B; "
"available memory size: %lu B;\n",
#ifdef LINUX
(PRUint64)total_cache_size, (PRUint64)li->li_dbcachesize, availpages * pagesize
#else
(PRUint64)total_cache_size, (PRUint64)li->li_dbcachesize, memsize
#endif
);
/* autosizing dbCache and entryCache */
} else if (li->li_cache_autosize > 0) {
msg = "This can be corrected by altering the values of nsslapd-cache-autosize, nsslapd-cache-autosize-split and nsslapd-dncachememsize\n";
zone_pages = (li->li_cache_autosize * pages) / 100;
zone_size = zone_pages * pagesize;
/* This is how much we "might" use, lets check it's sane. */
/* In the case it is not, this will *reduce* the allocation */
issane = util_is_cachesize_sane(&zone_size);
if (!issane) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start",
"Your autosized cache values have been reduced. Likely your nsslapd-cache-autosize percentage is too high.\n");
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "%s", msg);
}
/* It's valid, lets divide it up and set according to user prefs */
zone_pages = zone_size / pagesize;
db_pages = (li->li_cache_autosize_split * zone_pages) / 100;
entry_pages = (zone_pages - db_pages) / objset_size(li->li_instance_set);
/* We update this for the is-sane check below. */
total_cache_size = (zone_pages - db_pages) * pagesize;
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "cache autosizing. found %luk physical memory\n",
pages*(pagesize/1024));
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "cache autosizing. found %luk avaliable\n",
zone_pages*(pagesize/1024));
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "cache autosizing: db cache: %luk, "
"each entry cache (%d total): %luk\n",
db_pages*(pagesize/1024), objset_size(li->li_instance_set),
entry_pages*(pagesize/1024));
/* libdb allocates 1.25x the amount we tell it to,
* but only for values < 500Meg
* For the larger memory, the overhead is relatively small. */
cache_size_to_configure = (unsigned long)(db_pages * pagesize);
if (cache_size_to_configure < (500 * MEGABYTE)) {
cache_size_to_configure = (unsigned long)((db_pages * pagesize) / 1.25);
}
sprintf(s, "%lu", cache_size_to_configure);
ldbm_config_internal_set(li, CONFIG_DBCACHESIZE, s);
li->li_cache_autosize_ec = (unsigned long)entry_pages * pagesize;
for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) {
inst = (ldbm_instance *)object_get_data(inst_obj);
cache_set_max_entries(&(inst->inst_cache), -1);
cache_set_max_size(&(inst->inst_cache),
li->li_cache_autosize_ec, CACHE_TYPE_ENTRY);
/* We need to get each instances dncache size to add to the total */
/* Else we can't properly check the cache allocations below */
/* Trac 48831 exists to allow this to be auto-sized too ... */
total_cache_size += (PRUint64)cache_get_max_size(&(inst->inst_dncache));
}
}
/* autosizing importCache */
if (li->li_import_cache_autosize > 0) {
/* For some reason, -1 means 50 ... */
if (li->li_import_cache_autosize == -1) {
li->li_import_cache_autosize = 50;
}
import_pages = (li->li_import_cache_autosize * pages) / 100;
import_size = import_pages * pagesize;
issane = util_is_cachesize_sane(&import_size);
if (!issane) {
slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start",
"Your autosized import cache values have been reduced. "
"Likely your nsslapd-import-cache-autosize percentage is too high.\n");
}
/* We just accept the reduced allocation here. */
import_pages = import_size / pagesize;
slapi_log_err(SLAPI_LOG_NOTICE, "ldbm_back_start", "cache autosizing: import cache: %luk\n",
import_pages*(pagesize/1024));
sprintf(s, "%lu", (unsigned long)(import_pages * pagesize));
ldbm_config_internal_set(li, CONFIG_IMPORT_CACHESIZE, s);
}
}
}
/* Finally, lets check that the total result is sane. */
total_size = total_cache_size + (PRUint64)li->li_dbcachesize;
issane = util_is_cachesize_sane(&total_size);
if (!issane) {
/* Right, it's time to panic */
slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start",
"It is highly likely your memory configuration of all backends will EXCEED your systems memory.\n");
slapi_log_err(SLAPI_LOG_CRIT, "ldbm_back_start",
"In a future release this WILL prevent server start up. You MUST alter your configuration.\n");
slapi_log_err(SLAPI_LOG_CRIT,
"ldbm_back_start", "Total entry cache size: %lu B; "
"dbcache size: %lu B; "
"available memory size: %lu B; \n",
#ifdef LINUX
(PRUint64)total_cache_size, (PRUint64)li->li_dbcachesize, availpages * pagesize
#else
(PRUint64)total_cache_size, (PRUint64)li->li_dbcachesize, memsize
#endif
);
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "%s\n", msg);
/* WB 2016 - This should be UNCOMMENTED in a future release */
/* return SLAPI_FAIL_GENERAL; */
}
retval = check_db_version(li, &action);
if (0 != retval)
{
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "db version is not supported\n");
return SLAPI_FAIL_GENERAL;
}
if (action &
(DBVERSION_UPGRADE_3_4|DBVERSION_UPGRADE_4_4|DBVERSION_UPGRADE_4_5))
{
retval = dblayer_start(li,DBLAYER_CLEAN_RECOVER_MODE);
}
else
{
retval = dblayer_start(li,DBLAYER_NORMAL_MODE);
}
if (0 != retval) {
char *msg;
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Failed to init database, err=%d %s\n",
retval, (msg = dblayer_strerror( retval )) ? msg : "");
if (LDBM_OS_ERR_IS_DISKFULL(retval)) return return_on_disk_full(li);
else return SLAPI_FAIL_GENERAL;
}
/* Walk down the instance list, starting all the instances. */
retval = ldbm_instance_startall(li);
if (0 != retval) {
char *msg;
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Failed to start databases, err=%d %s\n",
retval, (msg = dblayer_strerror( retval )) ? msg : "");
if (LDBM_OS_ERR_IS_DISKFULL(retval)) return return_on_disk_full(li);
else {
if ((li->li_cache_autosize > 0) && (li->li_cache_autosize <= 100)) {
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_start", "Failed to allocate %lu byte dbcache. "
"Please reduce the value of %s and restart the server.\n",
li->li_dbcachesize, CONFIG_CACHE_AUTOSIZE);
}
return SLAPI_FAIL_GENERAL;
}
}
/* write DBVERSION file if one does not exist */
home_dir = dblayer_get_home_dir(li, NULL);
if (!dbversion_exists(li, home_dir))
{
dbversion_write (li, home_dir, NULL, DBVERSION_ALL);
}
/* this function is called every time new db is initialized */
/* currently it is called the 2nd time when changelog db is */
/* dynamically created. Code below should only be called once */
if (!initialized)
{
ldbm_compute_init();
initialized = 1;
}
/* initialize the USN counter */
ldbm_usn_init(li);
slapi_log_err(SLAPI_LOG_TRACE, "ldbm_back_start", "ldbm backend done starting\n");
return( 0 );
}
int
ldbm_back_dbverify( Slapi_PBlock *pb )
{
struct ldbminfo *li = NULL;
Object *inst_obj = NULL;
ldbm_instance *inst = NULL;
int verbose = 0;
int rval = 1;
int rval_main = 0;
char **instance_names = NULL;
char *dbdir = NULL;
slapi_log_err(SLAPI_LOG_TRACE, "ldbm_back_dbverify", "Verifying db files...\n");
slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_names);
slapi_pblock_get(pb, SLAPI_SEQ_TYPE, &verbose);
slapi_pblock_get(pb, SLAPI_PLUGIN_PRIVATE, &li);
slapi_pblock_get(pb, SLAPI_DBVERIFY_DBDIR, &dbdir);
ldbm_config_load_dse_info(li);
ldbm_config_internal_set(li, CONFIG_DB_TRANSACTION_LOGGING, "off");
/* no write needed; choose EXPORT MODE */
if (0 != dblayer_start(li, DBLAYER_EXPORT_MODE))
{
slapi_log_err(SLAPI_LOG_ERR, "ldbm_back_dbverify",
"dbverify: Failed to init database\n");
return rval;
}
/* server is up */
slapi_log_err(SLAPI_LOG_TRACE, "ldbm_back_dbverify", "server is up\n");
if (instance_names) /* instance is specified */
{
char **inp = NULL;
for (inp = instance_names; inp && *inp; inp++)
{
inst = ldbm_instance_find_by_name(li, *inp);
if (inst)
{
if (dbdir){
/* verifying backup */
slapi_ch_free_string(&inst->inst_parent_dir_name);
inst->inst_parent_dir_name = slapi_ch_strdup(dbdir);
}
rval_main |= dbverify_ext(inst, verbose);
}
else
{
rval_main |= 1; /* no such instance */
}
}
}
else /* all instances */
{
for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj;
inst_obj = objset_next_obj(li->li_instance_set, inst_obj))
{
inst = (ldbm_instance *)object_get_data(inst_obj);
/* check if an import/restore is already ongoing... */
if (instance_set_busy(inst) != 0)
{
/* standalone, only. never happens */
slapi_log_err(SLAPI_LOG_WARNING, "ldbm_back_dbverify",
"Backend '%s' is already in the middle of "
"another task and cannot be disturbed.\n",
inst->inst_name);
continue; /* skip this instance and go to the next*/
}
if (dbdir){
/* verifying backup */
slapi_ch_free_string(&inst->inst_parent_dir_name);
inst->inst_parent_dir_name = slapi_ch_strdup(dbdir);
}
rval_main |= dbverify_ext(inst, verbose);
}
}
/* close the database down again */
rval = dblayer_post_close(li, DBLAYER_EXPORT_MODE);
if (0 != rval)
{
slapi_log_err(SLAPI_LOG_ERR,
"ldbm_back_dbverify", "Failed to close database\n");
}
return rval_main;
}
