/** \brief Initialize a semaphore.
* \param key Semaphore key to use.
* \return The freshly allocated semaphore handle. */
static int
ketama_sem_init( key_t key )
{
if (sem_ids == NULL) {
init_sem_id_tracker();
}
int sem_set_id;
sem_set_id = semget( key, 1, 0 );
track_sem_id(sem_set_id);
if ( sem_set_id == -1 )
{
// create a semaphore set with ID SEM_ID
sem_set_id = semget( key, 1, IPC_CREAT | 0666 );
track_sem_id(sem_set_id);
if ( sem_set_id == -1 )
{
strcpy( k_error, "Could not open semaphore!" );
return 0;
}
ketama_sem_unlock( sem_set_id );
}
return sem_set_id;
}
/** \brief Locks the semaphore only after it is unlocked by another process.
* \param sem_set_id The semaphore handle that you want to lock. */
static void
ketama_sem_safely_lock( int sem_set_id )
{
int sanity = 0;
while ( semctl( sem_set_id, 0, GETVAL, 0 ) == 2 )
{
// wait for the continuum creator to finish, but don't block others
usleep( 5 );
// if we are waiting for > 1 second, take drastic action:
if(++sanity > 200000)
{
usleep( rand()%50000 );
ketama_sem_unlock( sem_set_id );
break;
}
}
ketama_sem_lock( sem_set_id );
}
int
ketama_roll( ketama_continuum* contptr, char* filename )
{
strcpy( k_error, "" );
key_t key;
int shmid;
int *data;
int sem_set_id;
// setlogmask( LOG_UPTO ( LOG_NOTICE | LOG_ERR | LOG_INFO ) );
// openlog( "ketama", LOG_CONS | LOG_PID | LOG_NDELAY, LOG_LOCAL1 );
key = ftok( filename, 'R' );
if ( key == -1 )
{
sprintf( k_error, "Invalid filename specified: %s", filename );
return 0;
}
*contptr = malloc( sizeof( continuum ) );
(*contptr)->numpoints = 0;
(*contptr)->array = 0;
(*contptr)->modtime = 0;
sem_set_id = ketama_sem_init( key );
int sanity = 0;
while ( semctl( sem_set_id, 0, GETVAL, 0 ) == 2 )
{
// wait for the continuum creator to finish, but don't block others
usleep( 5 );
// if we are waiting for > 1 second, take drastic action:
if(++sanity > 1000000)
{
usleep( rand()%50000 );
ketama_sem_unlock( sem_set_id );
break;
}
}
time_t modtime = file_modtime( filename );
time_t* fmodtime = 0;
while ( !fmodtime || modtime != *fmodtime )
{
shmid = shmget( key, MC_SHMSIZE, 0 ); // read only attempt.
data = shmat( shmid, (void *)0, SHM_RDONLY );
if ( data == (void *)(-1) || (*contptr)->modtime != 0 )
{
ketama_sem_lock( sem_set_id );
// if ( (*contptr)->modtime == 0 )
// syslog( LOG_INFO, "Shared memory empty, creating and populating...\n" );
// else
// syslog( LOG_INFO, "Server definitions changed, reloading...\n" );
if ( !ketama_create_continuum( key, filename ) )
{
// strcpy( k_error, "Ketama_create_continuum() failed!" );
ketama_sem_unlock( sem_set_id );
return 0;
}
/* else
syslog( LOG_INFO, "ketama_create_continuum() successfully finished.\n" );*/
shmid = shmget( key, MC_SHMSIZE, 0 ); // read only attempt.
data = shmat( shmid, (void *)0, SHM_RDONLY );
ketama_sem_unlock( sem_set_id );
}
if ( data == (void *)(-1) )
{
strcpy( k_error, "Failed miserably to get pointer to shmemdata!" );
return 0;
}
(*contptr)->numpoints = *data;
(*contptr)->modtime = ++data;
(*contptr)->array = data + sizeof( void* );
fmodtime = (time_t*)( (*contptr)->modtime );
}
return 1;
}
int
reconstruct_continuum(key_t key, serverinfo* slist, int numservers, mcs* ring, int numpoints, unsigned long memory, time_t fmodtime) {
int shmid, sem_set_id;
continuum* data; // Pointer to shmem location
// sort the ring in ascending order of "point"
qsort( (void*) ring, numpoints, sizeof( mcs ), (compfn)ketama_compare );
// attempt to obtain the shared memory ID assigned to this key, and create a segment if it doesn't exist
shmid = shmget( key, MC_SHMSIZE, 0644 | IPC_CREAT );
if ( shmid == -1 ) {
snprintf( k_error, sizeof(k_error), "Ketama: reconstruct_continuum failed to get valid shared memory segment with errno: %d.\n", errno );
syslog1( LOG_INFO, k_error );
return 0;
}
// lock the semaphore to prevent other writing to the shared memory segment
sem_set_id = ketama_sem_init( key );
ketama_sem_safely_lock( sem_set_id );
int invalid_write = 0;
// create an attachment in virtual memory to the shared memory segment, and failout if that an error is returned
data = (continuum*) shmat( shmid, (void *)0, 0 );
if ( data == (void *)(-1) ) {
snprintf( k_error, sizeof(k_error), "Ketama: reconstruct_continuum failed to attach writable shared memory segment with errno: %d.\n", errno );
invalid_write = 1;
}
// verify that the mcs array can hold this many points
if ( sizeof(data->array) < sizeof(mcs) * numpoints ) {
snprintf( k_error, sizeof(k_error), "Ketama: reconstruct_continuum tried to exceed size of mcs array.\n" );
ketama_shmdt(data);
invalid_write = 1;
}
// verify that the serverinfo array can hold this many servers
if ( sizeof(data->slist) < sizeof(serverinfo) * numservers ) {
snprintf( k_error, sizeof(k_error), "Ketama: reconstruct_continuum tried to exceed size of servers array.\n" );
ketama_shmdt(data);
invalid_write = 1;
}
// gracefully failout if any of the preceeding checks discovered errors
if (invalid_write) {
syslog1( LOG_INFO, k_error );
free(ring);
free(slist);
ketama_sem_unlock( sem_set_id );
return 0;
}
// record the new data into the shared memory segment
data->numpoints = numpoints;
data->numservers = numservers;
data->memtotal = memory;
data->fmodtime = fmodtime;
memcpy( data->array, ring, sizeof( mcs ) * numpoints );
memcpy( data->slist, slist, sizeof( serverinfo ) * numservers );
syslog( LOG_INFO, "Ketama: copied mcs array into %p from %p and slist into %p from %p.\n", data->array, ring, data->slist, slist );
data->cont_version++;
data->cont_modtime = time(NULL);
// We detatch here because we will re-attach in read-only mode to actually use it.
if (ketama_shmdt(data) == -1) {
snprintf( k_error, sizeof(k_error), "Ketama: reconstruct_continuum failed to detatch from shared memory with errno: %d.\n", errno );
syslog1( LOG_INFO, k_error );
}
// unlock the semaphore
ketama_sem_unlock( sem_set_id );
// clean up
free(ring);
free(slist);
return 1;
}
/** \brief Generates the continuum of servers (each server as many points on a circle).
* \param key Shared memory key for storing the newly created continuum.
* \param filename Server definition file, which will be parsed to create this continuum.
* \param contptr The value of this pointer will contain the retrieved continuum resource.
* \return 0 on failure, 1 on success. */
static int
ketama_create_continuum(key_t key, char* filename, ketama_continuum* contptr)
{
int shmid, sem_set_id;
int numservers = 0;
unsigned long memory;
serverinfo* slist;
continuum* data;
// if filename is not a user-specific key, init a shared memory segment from the pathname, if it's valid
if (strncmp(filename, "key:", 4) != 0) {
// get the list of servers from the provided file
slist = read_server_definitions( filename, &numservers, &memory );
// Check numservers first; if it is zero then there is no error message and we need to set one.
if ( numservers < 1 ) {
snprintf( k_error, sizeof(k_error), "Ketama: No valid server definitions in file %s", filename );
syslog1( LOG_INFO, k_error );
return 0;
} else if ( slist == NULL ) {
// read_server_definitions must've set error message.
return 0;
}
// log the sucessful file read
syslog( LOG_INFO, "Ketama: Server definitions read: %u servers, total memory: %lu.\n", numservers, memory );
time_t fmodtime = file_modtime( filename );
// attempt to load the continuum and log an error if this fails
if ( !load_continuum( key, slist, numservers, memory, fmodtime ) ) {
snprintf( k_error, sizeof(k_error), "Ketama: ketama_create_continuum failed to load the continuum.\n" );
syslog1( LOG_INFO, k_error );
return 0;
}
}
// attempt to obtain the shared memory ID assigned to this key, and create a segment if it doesn't exist
shmid = shmget( key, MC_SHMSIZE, 0644 | IPC_CREAT );
if ( shmid == -1 ) {
snprintf( k_error, sizeof(k_error), "Ketama: ketama_create_continuum failed to get valid shared memory segment with errno: %d.\n", errno );
syslog1( LOG_INFO, k_error );
return 0;
}
// lock the semaphore to prevent other writing to the shared memory segment
sem_set_id = ketama_sem_init( key );
ketama_sem_safely_lock( sem_set_id );
// create an attachment in virtual memory to the shared memory segment, and failout if that an error is returned
data = (continuum*) shmat( shmid, (void *)0, 0 );
if ( data == (void *)(-1) ) {
snprintf( k_error, sizeof(k_error), "Ketama: ketama_create_continuum failed to attach writable shared memory segment with errno: %d.\n", errno );
syslog1( LOG_INFO, k_error );
ketama_sem_unlock( sem_set_id );
return 0;
}
// if filename is not a path, init an empty shared memory segment, otherwise just record the filename
if (strncmp(filename, "key:", 4) == 0) {
data->fmodtime = 0;
data->cont_version = 1;
data->cont_modtime = time(NULL);
}
snprintf(data->cont_filename, sizeof(data->cont_filename), "%s", filename);
// attempt to detach the shared memory segment and throw an error if one is received
if (ketama_shmdt(data) == -1) {
snprintf( k_error, sizeof(k_error), "Ketama: ketama_create_continuum failed to detatch from shared memory with errno: %d.\n", errno );
syslog1( LOG_INFO, k_error );
}
// unlock the semaphore
ketama_sem_unlock( sem_set_id );
// rerun ketama_roller with the roller flag on to validate the loaded continuum
return ketama_roller( contptr, filename, 1);
}
