Beispiel #1
0
static int detect_protocol(void)
{
	confdb_handle_t handle;
	hdb_handle_t totem_handle;
	char key_value[256];
	size_t value_len;
	int rv, proto = -1;
	confdb_callbacks_t callbacks = {
		.confdb_key_change_notify_fn = NULL,
		.confdb_object_create_change_notify_fn = NULL,
		.confdb_object_delete_change_notify_fn = NULL
	};

	rv = confdb_initialize(&handle, &callbacks);
	if (rv != CS_OK) {
		log_error("confdb_initialize error %d", rv);
		return -1; 
	}

	rv = confdb_object_find_start(handle, OBJECT_PARENT_HANDLE);
	if (rv != CS_OK) {
		log_error("confdb_object_find_start error %d", rv);
		goto out;
	}

	rv = confdb_object_find(handle, OBJECT_PARENT_HANDLE,
				"totem", strlen("totem"), &totem_handle);
	if (rv != CS_OK) {
		log_error("confdb_object_find error %d", rv);
		goto out;
	}

	rv = confdb_key_get(handle, totem_handle,
			    "rrp_mode", strlen("rrp_mode"),
			    key_value, &value_len);
	if (rv != CS_OK) {
		log_error("confdb_key_get error %d", rv);
		goto out;
	}

	key_value[value_len] = '\0';
	log_debug("totem/rrp_mode = '%s'", key_value);

	if (!strcmp(key_value, "none"))
		proto = PROTO_TCP;
	else
		proto = PROTO_SCTP;
 out:
	confdb_finalize(handle);
	return proto;
}
Beispiel #2
0
static int
get_config_opt(confdb_handle_t config,
               hdb_handle_t object_handle, const char *key, char **value, const char *fallback)
{
    size_t len = 0;
    char *env_key = NULL;
    const char *env_value = NULL;
    char buffer[256];

    if (*value) {
        free(*value);
        *value = NULL;
    }

    if (object_handle > 0) {
        if (CS_OK == confdb_key_get(config, object_handle, key, strlen(key), &buffer, &len)) {
            *value = strdup(buffer);
        }
    }

    if (*value) {
        crm_info("Found '%s' for option: %s", *value, key);
        return 0;
    }

    env_key = crm_concat("HA", key, '_');
    env_value = getenv(env_key);
    free(env_key);

    if (*value) {
        crm_info("Found '%s' in ENV for option: %s", *value, key);
        *value = strdup(env_value);
        return 0;
    }

    if (fallback) {
        crm_info("Defaulting to '%s' for option: %s", fallback, key);
        *value = strdup(fallback);

    } else {
        crm_info("No default for option: %s", key);
    }

    return -1;
}
/*
 * Caller should free the returned string
 */
static const char *get_quorum_type(void)
{
	const char *qtype = NULL;
	int result;
	char buf[255];
	size_t namelen = sizeof(buf);
	hdb_handle_t quorum_handle;
	confdb_handle_t confdb_handle;
	confdb_callbacks_t callbacks = {
        	.confdb_key_change_notify_fn = NULL,
        	.confdb_object_create_change_notify_fn = NULL,
        	.confdb_object_delete_change_notify_fn = NULL
	};

	if (confdb_initialize(&confdb_handle, &callbacks) != CS_OK) {
		errno = EINVAL;
		return NULL;
	}
        result = confdb_object_find_start(confdb_handle, OBJECT_PARENT_HANDLE);
	if (result != CS_OK)
		goto out;

        result = confdb_object_find(confdb_handle, OBJECT_PARENT_HANDLE, (void *)"quorum", strlen("quorum"), &quorum_handle);
        if (result != CS_OK)
		goto out;

        result = confdb_key_get(confdb_handle, quorum_handle, (void *)"provider", strlen("provider"), buf, &namelen);
        if (result != CS_OK)
		goto out;

	if (namelen >= sizeof(buf)) {
		namelen = sizeof(buf) - 1;
	}
	buf[namelen] = '\0';

	/* If strdup returns NULL then we just assume no quorum provider ?? */
	qtype = strdup(buf);

out:
	confdb_finalize(confdb_handle);
	return qtype;
}
Beispiel #4
0
int main (int argc, char *argv[]) {
	confdb_handle_t handle;
	int result;
	hdb_handle_t totem_handle;
	char key_value[256];
	size_t value_len;

	result = confdb_initialize (&handle, &callbacks);
	if (result != CS_OK) {
		printf ("Could not initialize Cluster Configuration Database API instance error %d\n", result);
		exit (1);
	}

	if (argv[1] && strcmp(argv[1], "write")==0)
		do_write_tests(handle);

	if (argv[1] && strcmp(argv[1], "reload")==0) {
		/* Test reload interface */
		result = confdb_reload(handle, 0, key_value, sizeof key_value);
		printf ("Try to reload the config (noflush): %d (should be 1)\n", result);

		result = confdb_reload(handle, 1, key_value, sizeof key_value);
		printf ("Try to reload the config (flush): %d (should be 1)\n", result);
	}

	/* Test iterators */
	print_config_tree(handle, OBJECT_PARENT_HANDLE, 0);

	/* Find "totem" and dump bits of it again, to test the direct APIs */
	result = confdb_object_find_start(handle, OBJECT_PARENT_HANDLE);
	if (result != CS_OK) {
		printf ("Could not start object_find %d\n", result);
		exit (1);
	}

	result = confdb_object_find(handle, OBJECT_PARENT_HANDLE, "totem", strlen("totem"), &totem_handle);
	if (result != CS_OK) {
		printf ("Could not object_find \"totem\": %d\n", result);
		exit (1);
	}

	result = confdb_key_get(handle, totem_handle, "version", strlen("version"), key_value, &value_len);
	if (result != CS_OK) {
		printf ("Could not get \"version\" key: %d\n", result);
		exit (1);
	}
	key_value[value_len] = '\0';
	printf("totem/version = '%s'\n", key_value);

	result = confdb_key_get(handle, totem_handle, "secauth", strlen("secauth"), key_value, &value_len);
	if (result != CS_OK) {
		printf ("Could not get \"secauth\" key: %d\n", result);
		exit (1);
	}
	key_value[value_len] = '\0';
	printf("totem/secauth = '%s'\n", key_value);

	/* Try a call that fails */
	result = confdb_key_get(handle, totem_handle, "grot", strlen("grot"), key_value, &value_len);
	printf ("Get \"grot\" key returned: %d (should fail)\n", result);

	result = confdb_finalize (handle);
	printf ("Finalize  result is %d (should be 1)\n", result);
	return (0);
}
Beispiel #5
0
/*
 * Test confdb integration + quit policy
 */
static int test8 (pid_t pid, pid_t old_pid, int test_n) {
	confdb_handle_t cdb_handle;
	cs_error_t err;
	hdb_handle_t res_handle, proc_handle, pid_handle;
	size_t value_len;
	uint64_t tstamp1, tstamp2;
	int32_t msec_diff;
	char key_value[256];
	unsigned int instance_id;
	char tmp_obj[PATH_MAX];
	confdb_value_types_t cdbtype;

	err = confdb_initialize (&cdb_handle, NULL);
	if (err != CS_OK) {
		printf ("Could not initialize Cluster Configuration Database API instance error %d. Test skipped\n", err);
		return (1);
	}

	printf ("%s test %d\n", __FUNCTION__, test_n);

	if (test_n == 2) {
		/*
		 * Object should not exist
		 */
		printf ("%s Testing if object exists (it shouldn't)\n", __FUNCTION__);

		err = confdb_object_find_start(cdb_handle, OBJECT_PARENT_HANDLE);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, OBJECT_PARENT_HANDLE, "resources", strlen("resources"), &res_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"resources\": %d.\n", err);
			return (2);
		}

		err = confdb_object_find_start(cdb_handle, res_handle);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, res_handle, "process", strlen("process"), &proc_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"process\": %d.\n", err);
			return (2);
		}

		if (snprintf (tmp_obj, sizeof (tmp_obj), "%s:%d", __progname, pid) >= sizeof (tmp_obj)) {
			snprintf (tmp_obj, sizeof (tmp_obj), "%d", pid);
		}

		err = confdb_object_find_start(cdb_handle, proc_handle);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, proc_handle, tmp_obj, strlen(tmp_obj), &pid_handle);
		if (err == CS_OK) {
			printf ("Could find object \"%s\": %d.\n", tmp_obj, err);
			return (2);
		}
	}

	if (test_n == 1 || test_n == 2) {
		printf ("%s: initialize\n", __FUNCTION__);
		err = sam_initialize (2000, SAM_RECOVERY_POLICY_QUIT | SAM_RECOVERY_POLICY_CONFDB);
		if (err != CS_OK) {
			fprintf (stderr, "Can't initialize SAM API. Error %d\n", err);
			return 2;
		}

		printf ("%s: register\n", __FUNCTION__);
		err = sam_register (&instance_id);
		if (err != CS_OK) {
			fprintf (stderr, "Can't register. Error %d\n", err);
			return 2;
		}

		err = confdb_object_find_start(cdb_handle, OBJECT_PARENT_HANDLE);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, OBJECT_PARENT_HANDLE, "resources", strlen("resources"), &res_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"resources\": %d.\n", err);
			return (2);
		}

		err = confdb_object_find_start(cdb_handle, res_handle);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, res_handle, "process", strlen("process"), &proc_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"process\": %d.\n", err);
			return (2);
		}

		if (snprintf (tmp_obj, sizeof (tmp_obj), "%s:%d", __progname, pid) >= sizeof (tmp_obj)) {
			snprintf (tmp_obj, sizeof (tmp_obj), "%d", pid);
		}

		err = confdb_object_find_start(cdb_handle, proc_handle);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, proc_handle, tmp_obj, strlen(tmp_obj), &pid_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"%s\": %d.\n", tmp_obj, err);
			return (2);
		}

		err = confdb_key_get(cdb_handle, pid_handle, "recovery", strlen("recovery"), key_value, &value_len);
		if (err != CS_OK) {
			printf ("Could not get \"recovery\" key: %d.\n", err);
			return (2);
		}

		if (value_len != strlen ("quit") || memcmp (key_value, "quit", value_len) != 0) {
			printf ("Recovery key \"%s\" is not \"watchdog\".\n", key_value);
			return (2);
		}

		err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
		if (err != CS_OK) {
			printf ("Could not get \"state\" key: %d.\n", err);
			return (2);
		}

		if (value_len != strlen ("stopped") || memcmp (key_value, "stopped", value_len) != 0) {
			printf ("State key is not \"stopped\".\n");
			return (2);
		}

		printf ("%s iid %d: start\n", __FUNCTION__, instance_id);
		err = sam_start ();
		if (err != CS_OK) {
			fprintf (stderr, "Can't start hc. Error %d\n", err);
			return 2;
		}

		err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
		if (err != CS_OK) {
			printf ("Could not get \"state\" key: %d.\n", err);
			return (2);
		}

		if (value_len != strlen ("running") || memcmp (key_value, "running", value_len) != 0) {
			printf ("State key is not \"running\".\n");
			return (2);
		}

		printf ("%s iid %d: stop\n", __FUNCTION__, instance_id);
		err = sam_stop ();
		if (err != CS_OK) {
			fprintf (stderr, "Can't stop hc. Error %d\n", err);
			return 2;
		}

		err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
		if (err != CS_OK) {
			printf ("Could not get \"state\" key: %d.\n", err);
			return (2);
		}

		if (value_len != strlen ("stopped") || memcmp (key_value, "stopped", value_len) != 0) {
			printf ("State key is not \"stopped\".\n");
			return (2);
		}

		printf ("%s iid %d: sleeping 5\n", __FUNCTION__, instance_id);
		sleep (5);

		err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
		if (err != CS_OK) {
			printf ("Could not get \"state\" key: %d.\n", err);
			return (2);
		}

		if (value_len != strlen ("stopped") || memcmp (key_value, "stopped", value_len) != 0) {
			printf ("State key is not \"stopped\".\n");
			return (2);
		}

		printf ("%s iid %d: start 2\n", __FUNCTION__, instance_id);
		err = sam_start ();
		if (err != CS_OK) {
			fprintf (stderr, "Can't start hc. Error %d\n", err);
			return 2;
		}

		err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
		if (err != CS_OK) {
			printf ("Could not get \"state\" key: %d.\n", err);
			return (2);
		}

		if (value_len != strlen ("running") || memcmp (key_value, "running", value_len) != 0) {
			printf ("State key is not \"running\".\n");
			return (2);
		}

		if (test_n == 2) {
			printf ("%s iid %d: sleeping 5. Should be killed\n", __FUNCTION__, instance_id);
			sleep (5);

			return (2);
		} else {
			printf ("%s iid %d: Test HC\n", __FUNCTION__, instance_id);
			err = sam_hc_send ();
			if (err != CS_OK) {
				fprintf (stderr, "Can't send hc. Error %d\n", err);
				return 2;
			}
			err = confdb_key_get_typed (cdb_handle, pid_handle, "last_updated", &tstamp1, &value_len, &cdbtype);
			if (err != CS_OK) {
				printf ("Could not get \"state\" key: %d.\n", err);
				return (2);
			}
			printf ("%s iid %d: Sleep 1\n", __FUNCTION__, instance_id);
			sleep (1);
			err = sam_hc_send ();
			if (err != CS_OK) {
				fprintf (stderr, "Can't send hc. Error %d\n", err);
				return 2;
			}
			sleep (1);
			err = confdb_key_get_typed (cdb_handle, pid_handle, "last_updated", &tstamp2, &value_len, &cdbtype);
			if (err != CS_OK) {
				printf ("Could not get \"state\" key: %d.\n", err);
				return (2);
			}
			msec_diff = (tstamp2 - tstamp1)/CS_TIME_NS_IN_MSEC;

			if (msec_diff < 500 || msec_diff > 2000) {
				printf ("Difference %d is not within <500, 2000> interval.\n", msec_diff);
				return (2);
			}

			printf ("%s iid %d: stop 2\n", __FUNCTION__, instance_id);
			err = sam_stop ();
			if (err != CS_OK) {
				fprintf (stderr, "Can't stop hc. Error %d\n", err);
				return 2;
			}

			err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
			if (err != CS_OK) {
				printf ("Could not get \"state\" key: %d.\n", err);
				return (2);
			}

			if (value_len != strlen ("stopped") || memcmp (key_value, "stopped", value_len) != 0) {
				printf ("State key is not \"stopped\".\n");
				return (2);
			}

			printf ("%s iid %d: exiting\n", __FUNCTION__, instance_id);
			return (0);
		}
	}

	if (test_n == 3) {
		printf ("%s Testing if status is failed\n", __FUNCTION__);

		/*
		 * Previous should be FAILED
		 */
		err = confdb_object_find_start(cdb_handle, OBJECT_PARENT_HANDLE);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, OBJECT_PARENT_HANDLE, "resources", strlen("resources"), &res_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"resources\": %d.\n", err);
			return (2);
		}

		err = confdb_object_find_start(cdb_handle, res_handle);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, res_handle, "process", strlen("process"), &proc_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"process\": %d.\n", err);
			return (2);
		}

		if (snprintf (tmp_obj, sizeof (tmp_obj), "%s:%d", __progname, pid) >= sizeof (tmp_obj)) {
			snprintf (tmp_obj, sizeof (tmp_obj), "%d", pid);
		}

		err = confdb_object_find_start(cdb_handle, proc_handle);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, proc_handle, tmp_obj, strlen(tmp_obj), &pid_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"%s\": %d.\n", tmp_obj, err);
			return (2);
		}

		err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
		if (err != CS_OK) {
			printf ("Could not get \"state\" key: %d.\n", err);
			return (2);
		}

		if (value_len != strlen ("failed") || memcmp (key_value, "failed", value_len) != 0) {
			printf ("State key is not \"failed\".\n");
			return (2);
		}

		return (0);
	}

	return (2);
}
Beispiel #6
0
/*
 * Test quorum
 */
static int test7 (void) {
	confdb_handle_t cdb_handle;
	cs_error_t err;
	hdb_handle_t quorum_handle;
	size_t value_len;
	char key_value[256];
	unsigned int instance_id;
	pthread_t kill_thread;

	err = confdb_initialize (&cdb_handle, NULL);
	if (err != CS_OK) {
		printf ("Could not initialize Cluster Configuration Database API instance error %d. Test skipped\n", err);
		return (1);
	}

	err = confdb_object_find_start(cdb_handle, OBJECT_PARENT_HANDLE);
	if (err != CS_OK) {
		printf ("Could not start object_find %d. Test skipped\n", err);
		return (1);
        }

	err = confdb_object_find(cdb_handle, OBJECT_PARENT_HANDLE, "quorum", strlen("quorum"), &quorum_handle);
	if (err != CS_OK) {
		printf ("Could not object_find \"quorum\": %d. Test skipped\n", err);
		return (1);
	}

	err = confdb_key_get(cdb_handle, quorum_handle, "provider", strlen("provider"), key_value, &value_len);
	if (err != CS_OK) {
		printf ("Could not get \"provider\" key: %d. Test skipped\n", err);
		return (1);
	}

        if (!(value_len - 1 == strlen ("testquorum") && memcmp (key_value, "testquorum", value_len - 1) == 0)) {
		printf ("Provider is not testquorum. Test skipped\n");
		return (1);
        }

	/*
	 * Set to not quorate
	 */
	err = confdb_key_create(cdb_handle, quorum_handle, "quorate", strlen("quorate"), "0", strlen("0"));
	if (err != CS_OK) {
		printf ("Can't create confdb key. Error %d\n", err);
		return (2);
	}

	printf ("%s: initialize\n", __FUNCTION__);
	err = sam_initialize (2000, SAM_RECOVERY_POLICY_QUORUM_RESTART);
	if (err != CS_OK) {
		fprintf (stderr, "Can't initialize SAM API. Error %d\n", err);
		return 2;
	}

	printf ("%s: register\n", __FUNCTION__);
	err = sam_register (&instance_id);
	if (err != CS_OK) {
		fprintf (stderr, "Can't register. Error %d\n", err);
		return 2;
	}

	if (instance_id == 1) {
		/*
		 * Sam start should block forever, but 10s for us should be enough
		 */
		pthread_create (&kill_thread, NULL, test7_thread, NULL);

		printf ("%s iid %d: start - should block forever (waiting 5s)\n", __FUNCTION__, instance_id);
		err = sam_start ();
		if (err != CS_OK) {
			fprintf (stderr, "Can't start hc. Error %d\n", err);
			return 2;
		}

		printf ("%s iid %d: wasn't killed\n", __FUNCTION__, instance_id);
		return (2);
	}

	if (instance_id == 2) {
		/*
		 * Set to quorate
		 */
		err = confdb_key_create(cdb_handle, quorum_handle, "quorate", strlen("quorate"), "1", strlen("1"));
		if (err != CS_OK) {
			printf ("Can't create confdb key. Error %d\n", err);
			return (2);
		}

		printf ("%s iid %d: start\n", __FUNCTION__, instance_id);
		err = sam_start ();
		if (err != CS_OK) {
			fprintf (stderr, "Can't start hc. Error %d\n", err);
			return 2;
		}

		/*
		 * Set corosync unquorate
		 */
		err = confdb_key_create(cdb_handle, quorum_handle, "quorate", strlen("quorate"), "0", strlen("0"));
		if (err != CS_OK) {
			printf ("Can't create confdb key. Error %d\n", err);
			return (2);
		}

		printf ("%s iid %d: sleep 3\n", __FUNCTION__, instance_id);
		sleep (3);

		printf ("%s iid %d: wasn't killed\n", __FUNCTION__, instance_id);
		return (2);
	}

	if (instance_id == 3) {
		return (0);
	}

	return (2);
}
Beispiel #7
0
/*
 * Test confdb integration + restart policy
 */
static int test9 (pid_t pid, pid_t old_pid, int test_n) {
	confdb_handle_t cdb_handle;
	cs_error_t err;
	hdb_handle_t res_handle, proc_handle, pid_handle;
	size_t value_len;
	char key_value[256];
	unsigned int instance_id;
	char tmp_obj[PATH_MAX];

	err = confdb_initialize (&cdb_handle, NULL);
	if (err != CS_OK) {
		printf ("Could not initialize Cluster Configuration Database API instance error %d. Test skipped\n", err);
		return (1);
	}

	printf ("%s test %d\n", __FUNCTION__, test_n);

	if (test_n == 1) {
		printf ("%s: initialize\n", __FUNCTION__);
		err = sam_initialize (2000, SAM_RECOVERY_POLICY_RESTART | SAM_RECOVERY_POLICY_CONFDB);
		if (err != CS_OK) {
			fprintf (stderr, "Can't initialize SAM API. Error %d\n", err);
			return 2;
		}

		printf ("%s: register\n", __FUNCTION__);
		err = sam_register (&instance_id);
		if (err != CS_OK) {
			fprintf (stderr, "Can't register. Error %d\n", err);
			return 2;
		}
		printf ("%s: iid %d\n", __FUNCTION__, instance_id);

		if (instance_id < 3) {
			err = confdb_object_find_start(cdb_handle, OBJECT_PARENT_HANDLE);
			if (err != CS_OK) {
				printf ("Could not start object_find %d.\n", err);
				return (2);
			}

			err = confdb_object_find(cdb_handle, OBJECT_PARENT_HANDLE, "resources", strlen("resources"),
			    &res_handle);
			if (err != CS_OK) {
				printf ("Could not object_find \"resources\": %d.\n", err);
				return (2);
			}

			err = confdb_object_find_start(cdb_handle, res_handle);
			if (err != CS_OK) {
				printf ("Could not start object_find %d.\n", err);
				return (2);
			}

			err = confdb_object_find(cdb_handle, res_handle, "process", strlen("process"), &proc_handle);
			if (err != CS_OK) {
				printf ("Could not object_find \"process\": %d.\n", err);
				return (2);
			}

			if (snprintf (tmp_obj, sizeof (tmp_obj), "%s:%d", __progname, pid) >= sizeof (tmp_obj)) {
				snprintf (tmp_obj, sizeof (tmp_obj), "%d", pid);
			}

			err = confdb_object_find_start(cdb_handle, proc_handle);
			if (err != CS_OK) {
				printf ("Could not start object_find %d.\n", err);
				return (2);
			}

			err = confdb_object_find(cdb_handle, proc_handle, tmp_obj, strlen(tmp_obj), &pid_handle);
			if (err != CS_OK) {
				printf ("Could not object_find \"%s\": %d.\n", tmp_obj, err);
				return (2);
			}

			err = confdb_key_get(cdb_handle, pid_handle, "recovery", strlen("recovery"), key_value, &value_len);
			if (err != CS_OK) {
				printf ("Could not get \"recovery\" key: %d.\n", err);
				return (2);
			}

			if (value_len != strlen ("restart") || memcmp (key_value, "restart", value_len) != 0) {
				printf ("Recovery key \"%s\" is not \"restart\".\n", key_value);
				return (2);
			}

			err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
			if (err != CS_OK) {
				printf ("Could not get \"state\" key: %d.\n", err);
				return (2);
			}

			if (value_len != strlen ("stopped") || memcmp (key_value, "stopped", value_len) != 0) {
				printf ("State key is not \"stopped\".\n");
				return (2);
			}

			printf ("%s iid %d: start\n", __FUNCTION__, instance_id);
			err = sam_start ();
			if (err != CS_OK) {
				fprintf (stderr, "Can't start hc. Error %d\n", err);
				return 2;
			}

			err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
			if (err != CS_OK) {
				printf ("Could not get \"state\" key: %d.\n", err);
				return (2);
			}

			if (value_len != strlen ("running") || memcmp (key_value, "running", value_len) != 0) {
				printf ("State key is not \"running\".\n");
				return (2);
			}

			printf ("%s iid %d: waiting for kill\n", __FUNCTION__, instance_id);
			sleep (10);

			return (2);
		}

		if (instance_id == 3) {
			printf ("%s iid %d: mark failed\n", __FUNCTION__, instance_id);
			if (err != CS_OK) {
				fprintf (stderr, "Can't start hc. Error %d\n", err);
				return 2;
			}
			err = sam_mark_failed ();
			if (err != CS_OK) {
				fprintf (stderr, "Can't mark failed. Error %d\n", err);
				return 2;
			}

			sleep (10);

			return (2);
		}

		return (2);
	}

	if (test_n == 2) {
		printf ("%s Testing if status is failed\n", __FUNCTION__);

		/*
		 * Previous should be FAILED
		 */
		err = confdb_object_find_start(cdb_handle, OBJECT_PARENT_HANDLE);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, OBJECT_PARENT_HANDLE, "resources", strlen("resources"), &res_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"resources\": %d.\n", err);
			return (2);
		}

		err = confdb_object_find_start(cdb_handle, res_handle);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, res_handle, "process", strlen("process"), &proc_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"process\": %d.\n", err);
			return (2);
		}

		if (snprintf (tmp_obj, sizeof (tmp_obj), "%s:%d", __progname, pid) >= sizeof (tmp_obj)) {
			snprintf (tmp_obj, sizeof (tmp_obj), "%d", pid);
		}

		err = confdb_object_find_start(cdb_handle, proc_handle);
		if (err != CS_OK) {
			printf ("Could not start object_find %d.\n", err);
			return (2);
		}

		err = confdb_object_find(cdb_handle, proc_handle, tmp_obj, strlen(tmp_obj), &pid_handle);
		if (err != CS_OK) {
			printf ("Could not object_find \"%s\": %d.\n", tmp_obj, err);
			return (2);
		}

		err = confdb_key_get(cdb_handle, pid_handle, "state", strlen("state"), key_value, &value_len);
		if (err != CS_OK) {
			printf ("Could not get \"state\" key: %d.\n", err);
			return (2);
		}

		if (value_len != strlen ("failed") || memcmp (key_value, "failed", value_len) != 0) {
			printf ("State key is not \"failed\".\n");
			return (2);
		}

		return (0);
	}

	return (2);
}