//
// Grunt destructor.
//
Grunt::~Grunt()
{
// Remove completion queue, if any.
if ( (type != InvalidType) && !IsType(type, GenericVIType) && io_cq )
delete io_cq;
Size_Target_Array( 0 );
// Release grunt's I/O data buffers if they are in use.
if ( data_size )
#if defined(IOMTR_OSFAMILY_NETWARE)
NXMemFree( read_data );
NXMemFree( write_data );
#elif defined(IOMTR_OSFAMILY_UNIX)
free( read_data );
free( write_data );
#elif defined(IOMTR_OSFAMILY_WINDOWS)
VirtualFree( read_data, 0, MEM_RELEASE );
VirtualFree( write_data, 0, MEM_RELEASE );
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
Free_Transaction_Arrays();
}
//
// Destructor
//
Manager::~Manager()
{
int g;
delete [] m_pVersionString;
delete [] m_pVersionStringWithDebug;
prt->Close();
delete prt;
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_SOLARIS)
if(data != NULL)
free(data);
if(swap_devices != NULL)
free(swap_devices);
#elif defined(IOMTR_OS_NETWARE)
if(data != NULL)
NXMemFree(data);
if(swap_devices != NULL)
NXMemFree(swap_devices);
#elif defined(IOMTR_OS_WIN32) || defined(IOMTR_OS_WIN64)
VirtualFree( data, 0, MEM_RELEASE );
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
for ( g = 0; g < grunt_count; g++ )
delete grunts[g];
}
//
// Free all memory associated with related I/O request arrays.
//
void Grunt::Free_Transaction_Arrays()
{
if ( trans_slots )
#if defined(IOMTR_OSFAMILY_NETWARE)
NXMemFree( trans_slots );
#elif defined(IOMTR_OSFAMILY_UNIX) || defined(IOMTR_OSFAMILY_WINDOWS)
free( trans_slots );
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
if ( available_trans_queue )
#if defined(IOMTR_OSFAMILY_NETWARE)
NXMemFree( available_trans_queue );
#elif defined(IOMTR_OSFAMILY_UNIX) || defined(IOMTR_OSFAMILY_WINDOWS)
free( available_trans_queue );
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
total_trans_slots = 0;
cur_trans_slots = 0;
}
BOOL SetQueueSize(HANDLE cqid, int size)
{
// Allocate memory for size elements in the completion queue.
// and also for the parallel array of struct aiocb pointers.
struct IOCQ *this_cqid;
this_cqid = (struct IOCQ *)cqid;
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_SOLARIS)
this_cqid->element_list = (struct CQ_Element *)malloc(sizeof(CQ_Element) * size);
#elif defined(IOMTR_OS_NETWARE)
this_cqid->element_list = (struct CQ_Element *)NXMemAlloc(sizeof(CQ_Element) * size, 1);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
if (this_cqid->element_list == NULL) {
cout << "memory allocation failed" << endl;
return (FALSE);
}
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_SOLARIS)
this_cqid->aiocb_list = (struct aiocb64 **)malloc(sizeof(struct aiocb64 *) * size);
#elif defined(IOMTR_OS_NETWARE)
this_cqid->aiocb_list = (struct aiocb64 **)NXMemAlloc(sizeof(struct aiocb64 *) * size, 1);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
if (this_cqid->aiocb_list == NULL) {
cout << "memory allocation failed" << endl;
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_SOLARIS)
free(this_cqid->element_list);
#elif defined(IOMTR_OS_NETWARE)
NXMemFree(this_cqid->element_list);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
return (FALSE);
}
memset(this_cqid->element_list, 0, sizeof(struct CQ_Element) * size);
memset(this_cqid->aiocb_list, 0, sizeof(struct aiocb64 *) * size);
this_cqid->size = size;
this_cqid->last_freed = -1;
this_cqid->position = 0;
#ifdef _DEBUG
cout << "allocated a completion queue of size " << size << " for handle : " << this_cqid << endl;
#endif
return (TRUE);
}
BOOL CQAIO::SetQueueSize(int size)
{
struct IOCQ *this_cqid = (struct IOCQ *)completion_queue;
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_SOLARIS)
this_cqid->element_list = (struct CQ_Element *)malloc(sizeof(CQ_Element) * size);
#elif defined(IOMTR_OS_NETWARE)
this_cqid->element_list = (struct CQ_Element *)NXMemAlloc(sizeof(CQ_Element) * size, 1);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
if (this_cqid->element_list == NULL) {
cout << "memory allocation failed." << endl;
return (FALSE);
}
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_SOLARIS)
this_cqid->aiocb_list = (struct aiocb64 **)malloc(sizeof(struct aiocb64 *) * size);
#elif defined(IOMTR_OS_NETWARE)
this_cqid->aiocb_list = (struct aiocb64 **)NXMemAlloc(sizeof(struct aiocb64 *) * size, 1);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
if (this_cqid->aiocb_list == NULL) {
cout << "memory allocation failed." << endl;
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_SOLARIS)
free(this_cqid->element_list);
#elif defined(IOMTR_OS_NETWARE)
NXMemFree(this_cqid->element_list);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
return (FALSE);
}
this_cqid->size = size;
memset(this_cqid->aiocb_list, 0, sizeof(struct aiocb64 *) * size);
memset(this_cqid->element_list, 0, sizeof(struct CQ_Element) * size);
#ifdef _DEBUG
cout << "allocated a completion queue of size " << size << " for handle : " << this_cqid << endl;
#endif
return (TRUE);
}
//
// Preparing a disk for access by a worker thread. The disk must have been
// previously initialized.
//
void Grunt::Prepare_Disk( int disk_id )
{
void *buffer = NULL;
DWORD buffer_size;
DWORDLONG prepare_offset = 0;
TargetDisk *disk = (TargetDisk *) targets[disk_id];
critical_error = FALSE;
// Allocate a large (64k for 512 byte sector size) buffer for the preparation.
buffer_size = disk->spec.disk_info.sector_size * 128;
#if defined(IOMTR_OSFAMILY_NETWARE)
NXMemFree( buffer );
errno = 0;
if ( !(buffer = NXMemAlloc(buffer_size, 1) ))
#elif defined(IOMTR_OSFAMILY_UNIX)
free( buffer );
errno = 0;
if ( !(buffer = valloc(buffer_size) ))
#elif defined(IOMTR_OSFAMILY_WINDOWS)
VirtualFree( buffer, 0, MEM_RELEASE );
if ( !(buffer = VirtualAlloc( NULL, buffer_size, MEM_COMMIT, PAGE_READWRITE )) )
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
{
cout << "*** Could not allocate buffer to prepare disk." << endl;
critical_error = TRUE;
InterlockedDecrement( (long *) ¬_ready );
return;
}
// Open the disk for preparation.
#if defined(IOMTR_OSFAMILY_NETWARE) || defined(IOMTR_OSFAMILY_UNIX)
// The disk::prepare() operation writes to a file iobw.tst till it uses up
// all the available disk space. Now, Solaris allows a file to be created
// with a (logical) size that is larger than the actual size of the file.
// Later, when an attempt is made to write to the unwritten portion of the
// file, Solaris attempts to expand the actual size on the disk to
// accomodate the new writes.
//
// The disk::prepare() throws up a problem here. Since we write in parallel
// to the the file, Solaris allows us to create an iobw.tst that is larger
// than the available space on the disk. A later write to the unfilled
// portion throws up an ENOSPC error. To avoid this problem, we use the
// O_APPEND flag which always sets the write offset to the eof.
if ( !disk->Open( &grunt_state, O_APPEND ) )
#elif defined(IOMTR_OSFAMILY_WINDOWS)
if ( !disk->Open( &grunt_state ) )
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
{
cout << "*** Could not open disk." << endl;
critical_error = TRUE;
}
else
{
// Prepare the disk, first with large block sizes, then with single sectors.
if ( !disk->Prepare( buffer, &prepare_offset, buffer_size, &grunt_state ) ||
!disk->Prepare( buffer, &prepare_offset, disk->spec.disk_info.sector_size, &grunt_state ) )
{
cout << "*** An error occurred while preparing the disk." << endl;
critical_error = TRUE;
}
disk->Close( NULL );
}
#if defined(IOMTR_OSFAMILY_NETWARE)
NXMemFree( buffer );
#elif defined(IOMTR_OSFAMILY_UNIX)
free( buffer );
#elif defined(IOMTR_OSFAMILY_WINDOWS)
VirtualFree( buffer, 0, MEM_RELEASE );
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
cout << " " << disk->spec.name << " done." << endl;
InterlockedDecrement( (long *) ¬_ready );
}
//
// Setting access specifications for worker. Also ensuring that a data buffer
// large enough to support the maximum requested transfer has been allocated.
// Note that Iometer will call Set_Access before testing starts to ensure that
// Dynamo can run the spec with the largest transfer request.
//
BOOL Grunt::Set_Access( const Test_Spec* spec )
{
// Check for idle spec.
if ((idle = ( spec->access[0].of_size == IOERROR)))
return TRUE;
access_spec.Initialize( &(spec->access[0]) );
// Allocate a data buffer large enough to support the maximum requested
// transfer. We do this only if the current buffer is too small and
// we're using per worker data buffers.
if ( data_size >= access_spec.max_transfer ) {
cout << "Grunt: Grunt data buffer size " << data_size << " >= "
<< access_spec.max_transfer << ", returning" << endl;
return TRUE;
} else if ( !data_size ) {
// We always want to use our own buffers, not the manager's
// buffer. This is due to a bug in some ServerWorks chipsets
// (confirmed on the HE-SL chipset) where performing both
// read and write operations to the same cache line can cause
// the PCI bridge (the CIOB5) to hang indefinitely until a
// third PCI bus request comes in.
//
// Using per-grunt buffers eliminates that problem, as you
// aren't thrashing on the same buffer for both read and
// write operations.
data_size = access_spec.max_transfer;
cout << "Grunt: Growing grunt data buffer from " << data_size << " to "
<< access_spec.max_transfer << endl;
}
// Allocating a larger buffer.
#if _DEBUG
cout << "Growing grunt data buffers from " << data_size << " to "
<< access_spec.max_transfer << endl;
#endif
#if defined(IOMTR_OSFAMILY_NETWARE)
if ( read_data ) {
NXMemFree( read_data );
}
errno = 0;
if ( !(read_data = NXMemAlloc(access_spec.max_transfer, 1) ))
#elif defined(IOMTR_OSFAMILY_UNIX)
if ( read_data ) {
free( read_data );
}
errno = 0;
if ( !(read_data = valloc(access_spec.max_transfer) ))
#elif defined(IOMTR_OSFAMILY_WINDOWS)
if ( read_data ) {
VirtualFree( read_data, 0, MEM_RELEASE );
}
if ( !(read_data = VirtualAlloc(NULL, access_spec.max_transfer, MEM_COMMIT, PAGE_READWRITE)))
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
{
// Could not allocate a larger buffer. Signal failure.
cout << "*** Grunt could not allocate read data buffer for I/O transfers." << endl;
data_size = 0;
return FALSE;
}
#if defined(IOMTR_OSFAMILY_NETWARE)
if ( write_data ) {
NXMemFree( write_data );
}
errno = 0;
if ( !(write_data = NXMemAlloc(access_spec.max_transfer, 1) ))
#elif defined(IOMTR_OSFAMILY_UNIX)
if ( write_data ) {
free( write_data );
}
errno = 0;
if ( !(write_data = valloc(access_spec.max_transfer) ))
#elif defined(IOMTR_OSFAMILY_WINDOWS)
if ( write_data ) {
VirtualFree( write_data, 0, MEM_RELEASE );
}
if ( !(write_data = VirtualAlloc(NULL, access_spec.max_transfer, MEM_COMMIT, PAGE_READWRITE)))
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
{
// Could not allocate a larger buffer. Signal failure.
cout << "*** Grunt could not allocate write data buffer for I/O transfers." << endl;
data_size = 0;
return FALSE;
}
data_size = access_spec.max_transfer;
return TRUE;
}
//
// Setting the size of the target array to hold the number and type of
// targets specified. If the requested number of targets is 0, the
// array will be freed.
//
BOOL Grunt::Size_Target_Array( int count, const Target_Spec *target_specs )
{
int i;
// Free all current targets. This is needed in case the newer targets
// are of a different type, even if we have the same number of targets.
for ( i = 0; i < target_count; i++ )
delete targets[i];
target_count = 0;
// Reset the grunt's target type.
type = InvalidType;
// Release the memory if everything is being freed.
if ( !count || !target_specs )
{
#if defined(IOMTR_OSFAMILY_NETWARE)
NXMemFree( targets );
#elif defined(IOMTR_OSFAMILY_UNIX)
free( targets );
#elif defined(IOMTR_OSFAMILY_WINDOWS)
free( targets ); // TODO: Check if VirtualFree() is not needed here.
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
targets = NULL;
return TRUE;
}
// Allocate enough pointers to refer to all targets.
#if defined(IOMTR_OSFAMILY_NETWARE)
targets = (Target**)NXMemRealloc( targets, sizeof(Target*) * count, 1 );
#elif defined(IOMTR_OSFAMILY_UNIX) || defined(IOMTR_OSFAMILY_WINDOWS)
targets = (Target**)realloc( targets, sizeof(Target*) * count );
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
if ( !targets )
return FALSE;
// Create the requested number of targets.
for ( i = 0; i < count; i++ )
{
type = (TargetType)(type | target_specs[i].type);
if ( IsType( target_specs[i].type, GenericDiskType ) )
targets[i] = new TargetDisk;
else if ( IsType( target_specs[i].type, GenericTCPType ) )
targets[i] = new TargetTCP;
#if defined(IOMTR_SETTING_VI_SUPPORT)
else if ( IsType( target_specs[i].type, GenericVIType ) )
targets[i] = new TargetVI;
#endif // IOMTR_SETTING_VI_SUPPORT
if ( !targets[i] )
return FALSE;
}
target_count = count;
return TRUE;
}
//
// Setting access specifications for next test.
// Note that Iometer will call Set_Access before testing starts to ensure that
// Dynamo can run the spec with the largest transfer request.
//
BOOL Manager::Set_Access( int target, const Test_Spec *spec )
{
int g; // loop control variable
// Recursively assign all workers the same access specification.
if ( target == ALL_WORKERS )
{
cout << "All workers running Access Spec: " << spec->name << endl;
for ( g = 0; g < grunt_count; g++ )
{
if ( !Set_Access( g, spec ) )
return FALSE;
}
return TRUE;
}
cout << "Worker " << target << " running Access Spec: " << spec->name << endl;
// If the grunt could not set the access spec properly, return.
// The grunt may not have been able to grow its data buffer.
if ( !grunts[target]->Set_Access( spec ) )
return FALSE;
// If the grunt is not using the manager's data buffer or the manager's
// buffer is already large enough, just return.
if ( grunts[target]->data_size ||
data_size >= grunts[target]->access_spec.max_transfer )
{
return TRUE;
}
// Grow the manager's data buffer and update all grunts using it.
#if _DEBUG
cout << "Growing manager data buffer from " << data_size << " to "
<< grunts[target]->access_spec.max_transfer << endl << flush;
#endif
// Align all data transfers on a page boundary. This will work for all disks
// with sector sizes that divide evenly into the page size - which is always
// the case.
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_SOLARIS)
free(data);
errno = 0;
if ( !(data = valloc(grunts[target]->access_spec.max_transfer) ))
#elif defined(IOMTR_OS_NETWARE)
NXMemFree(data);
errno = 0;
if ( !(data = NXMemAlloc(grunts[target]->access_spec.max_transfer, 1) ))
#elif defined(IOMTR_OS_WIN32) || defined(IOMTR_OS_WIN64)
VirtualFree( data, 0, MEM_RELEASE );
if ( !(data = VirtualAlloc( NULL, grunts[target]->access_spec.max_transfer,
MEM_COMMIT, PAGE_READWRITE )) )
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
{
// Could not allocate a larger buffer. Signal failure.
cout << "*** Manager could not allocate data buffer for I/O transfers."
<< endl << flush;
data_size = 0;
return FALSE;
}
data_size = grunts[target]->access_spec.max_transfer;
// Update all grunts using the manager's data buffer.
for ( g = 0; g < grunt_count; g++ )
{
if ( !grunts[g]->data_size )
{
grunts[g]->read_data = data;
grunts[g]->write_data = data;
}
}
return TRUE;
}
//
// CloseHandle() has a slightly different interface from the NT call. It takes an
// additional input parameter to determine the object type. The object can be either
// a FILE_ELEMENT or a CQ_ELEMENT.
//
// CloseHandle() closes the file handle or the completion queue handle and frees all
// the allocated memory. It does an additional check to ensure that all the queued
// Asynch I/Os that have not yet completed are cancelled before actually closing the
// handle. This helps clean up the kernel queues of any pending requests.
//
// Although it takes longer, this is acceptable since the code is not in the performance
// critical region.
//
BOOL CloseHandle(HANDLE object, int object_type)
{
struct File *filep;
struct IOCQ *cqid;
int retval, i;
#ifdef _DEBUG
cout << "CloseHandle() freeing : handle = " << object << " objecttype = " << object_type << endl;
#endif
switch (object_type) {
case FILE_ELEMENT:
filep = (struct File *)object;
cqid = filep->iocq;
// cancel any pending aio requests.
retval = aio_cancel64(filep->fd, NULL);
while (retval == AIO_NOTCANCELED) {
retval = aio_cancel64(filep->fd, NULL);
}
if (cqid != NULL && cqid->element_list != NULL && cqid->aiocb_list != NULL) {
for (i = 0; i < cqid->size; i++) {
if (cqid->element_list[i].aiocbp.aio_fildes != filep->fd)
continue;
// We are not interested in the return values of aio_error() and aio_return().
// only have to dequeue all the requests.
if (!cqid->aiocb_list[i])
continue;
retval = aio_error64(cqid->aiocb_list[i]);
retval = aio_return64(cqid->aiocb_list[i]);
}
}
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_SOLARIS)
close(filep->fd);
#elif defined(IOMTR_OS_NETWARE)
if (IsType(filep->type, LogicalDiskType))
NXClose(filep->fd);
else if (IsType(filep->type, PhysicalDiskType))
MM_ReleaseIOObject(filep->fd);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
break;
case CQ_ELEMENT:
cqid = (struct IOCQ *)object;
// cancel any pending aio requests.
for (i = 0; i < cqid->size; i++) {
if (!cqid->aiocb_list[i])
continue;
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_NETWARE)
/*
* In Linux, you crash (!) if the aiocpb isn't in your queue. :-(
* This code seems to occasionally do this...so I just cancel all
* AIOs for the queue, thus avoiding the problem of cancelling a
* message not in the queue.
*/
retval = aio_cancel64(cqid->element_list[i].aiocbp.aio_fildes, NULL);
#elif defined(IOMTR_OS_SOLARIS)
retval = aio_cancel64(cqid->element_list[i].aiocbp.aio_fildes, cqid->aiocb_list[i]);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
if (retval == AIO_NOTCANCELED) {
retval = aio_error64(cqid->aiocb_list[i]);
retval = aio_return64(cqid->aiocb_list[i]);
}
}
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_SOLARIS)
free(cqid->aiocb_list);
#elif defined(IOMTR_OS_NETWARE)
NXMemFree(cqid->aiocb_list);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
// Something strange here. If I free the element_list, the next round
// of aio_write() and aio_read() calls fail. If I dont free this, then they
// succeed. But then, there is a memory leak equal to the max number of outstanding
// I/Os * sizeof(CQ_Element). Does that mean that the above aio_cancel() calls
// are broken ??? It should be mentioned here that the element_list holds the
// actual aiocb structures.
//
// It suddenly seems to be working now.
// Remember to turn this "free" off when you hit the problem again.
// NEED TO LOOK INTO THIS.
#if defined(IOMTR_OS_LINUX) || defined(IOMTR_OS_OSX) || defined(IOMTR_OS_SOLARIS)
free(cqid->element_list);
free(cqid);
#elif defined(IOMTR_OS_NETWARE)
NXMemFree(cqid->element_list);
NXMemFree(cqid);
#else
#warning ===> WARNING: You have to do some coding here to get the port done!
#endif
break;
default:
break;
}
return (TRUE);
}
