int init_zeromq_pool(struct zeromq_pool * zpool){
WRITE_FMT_LOG(LOG_DEBUG, "%p", zpool);
if ( !zpool ) return ERR_BAD_ARG;
int err = 0;
/*create zmq context, it should be destroyed in zeromq_term; all opened sockets should be closed
*before finishing, either zmq_term will wait in internal loop for completion of all I/O requests*/
zpool->context = zmq_init(1);
if ( zpool->context ){
zpool->count_max=ESOCKF_ARRAY_GRANULARITY;
/*allocated memory for array should be free at the zeromq_term */
zpool->sockf_array = malloc(zpool->count_max * sizeof(struct sock_file_t));
if ( zpool->sockf_array ) {
memset(zpool->sockf_array, '\0', zpool->count_max*sizeof(struct sock_file_t));
for (int i=0; i < zpool->count_max; i++)
zpool->sockf_array[i].unused = 1;
err = ERR_OK;
}
else{
/*no memory allocated
* This code section can't be covered by unit tests because it's not depends from zpool parameter;
* sockf_array member is completely setups internally*/
WRITE_FMT_LOG(LOG_ERR, "%d bytes alloc error", (int)sizeof(struct sock_file_t));
err = ERR_NO_MEMORY;
}
}
else{
/* This code section can't be covered by unit tests because it's not depends from zpool parameter;
* it's only can be produced as unexpected zeromq error*/
WRITE_FMT_LOG(LOG_ERR, "zmq_init err %d, errno %d errtext %s\n", err, zmq_errno(), zmq_strerror(zmq_errno()));
err = ERR_ERROR;
}
return err;
}
/*i/o 2 files
*READ 1x struct request_data_t
*WRITE 1x char, reply
*READ 1x char is_complete flag, if 0
*WRITE 1x int, nodeid
*READ 1x char, reply
*WRITE 1x size_t, array_len
*READ 1x char, reply
*WRITE array_len x HistogramArrayItem, histograms array
* @return 1-should receive again, 0-complete request - it should not be listen again*/
int
read_requests_write_detailed_histograms( int fdr, int fdw, int nodeid, const BigArrayPtr source_array, int array_len){
int is_complete = 0;
do {
WRITE_FMT_LOG(LOG_DEBUG, "Reading from file %d detailed histograms request\n", fdr );fflush(0);
/*receive data needed to create histogram using step=1,
actually requested histogram should contains array items range*/
struct request_data_t received_histogram_request;
read_channel(fdr, (char*) &received_histogram_request, sizeof(struct request_data_t) );
read_channel(fdr, (char*) &is_complete, sizeof(is_complete) );
int histogram_len = 0;
//set to our offset, check it
int offset = min(received_histogram_request.first_item_index, array_len-1 );
int requested_length = received_histogram_request.last_item_index - received_histogram_request.first_item_index;
requested_length = min( requested_length, array_len - offset );
HistogramArrayPtr histogram = alloc_histogram_array_get_len( source_array, offset, requested_length, 1, &histogram_len );
size_t sending_array_len = histogram_len;
/*Response to request, entire reply contains requested detailed histogram*/
write_channel(fdw, (const char*)&nodeid, sizeof(int) );
write_channel(fdw, (const char*)&sending_array_len, sizeof(size_t) );
write_channel(fdw, (const char*)histogram, histogram_len*sizeof(HistogramArrayItem) );
free( histogram );
WRITE_FMT_LOG(LOG_DEBUG, "histograms wrote into file %d\n", fdw );
}while(!is_complete);
return is_complete;
}
/*READ 1x struct packet_data_t {type EPACKET_SEQUENCE_REQUEST}
*READ 1x struct request_data_t
* @param dst_pid destination process should receive ranges*/
int
read_range_request( int fdr, struct request_data_t* sequence ){
int len = 0;
WRITE_FMT_LOG(LOG_DEBUG, "reading range request from %d (EPACKET_SEQUENCE_REQUEST)\n", fdr);
struct packet_data_t t;
t.type = EPACKET_UNKNOWN;
read_channel( fdr, (char*)&t, sizeof(t) );
WRITE_FMT_LOG(LOG_DEBUG, "readed packet: type=%d, size=%d, src_node=%d\n", t.type, (int)t.size, t.src_nodeid );
if ( t.type == EPACKET_SEQUENCE_REQUEST )
{
struct request_data_t *data = NULL;
for ( int j=0; j < t.size; j++ ){
data = &sequence[j];
read_channel(fdr, (char*)data, sizeof(struct request_data_t));
WRITE_FMT_LOG(LOG_DEBUG, "recv range request %d %d %d\n", data->src_nodeid, data->first_item_index, data->last_item_index );
}
}
else{
WRITE_LOG(LOG_ERR,"channel_recv_sequences_request::packet Unknown\n");
assert(0);
}
return len;
}
int close_sockf(struct zeromq_pool* zpool, struct sock_file_t *sockf){
WRITE_FMT_LOG(LOG_DEBUG, "%p, %p", zpool, sockf);
int err = ERR_OK;
if ( !zpool || !sockf ) return ERR_BAD_ARG;
WRITE_FMT_LOG(LOG_DEBUG, "fd=%d", sockf->fs_fd);
int zmq_sock_is_used_twice = 0;
for (int i=0; i < zpool->count_max; i++){
if ( !zpool->sockf_array[i].unused && zpool->sockf_array[i].netw_socket == sockf->netw_socket )
zmq_sock_is_used_twice++;
}
if ( zmq_sock_is_used_twice > 1 ){
if ( sockf->tempbuf ){
free(sockf->tempbuf->buf), sockf->tempbuf->buf=NULL;
free(sockf->tempbuf), sockf->tempbuf = NULL;
}
WRITE_LOG(LOG_DEBUG, "dual direction zmq socket should be closed later");
}else if( sockf->netw_socket ){
WRITE_LOG(LOG_DEBUG, "zmq socket closing...");
int err = zmq_close( sockf->netw_socket );
sockf->netw_socket = NULL;
WRITE_FMT_LOG(LOG_DEBUG, "zmq_close status err %d, errno %d, status %s\n", err, zmq_errno(), zmq_strerror(zmq_errno()));
}
if ( ERR_OK == err )
return remove_sockf_from_array_by_fd(zpool, sockf->fs_fd );
else
return err;
}
ssize_t write_sockf(struct sock_file_t *sockf, const char *buf, size_t size){
int err = 0;
ssize_t wrote = 0;
WRITE_FMT_LOG(LOG_DEBUG, "%p, %p, %d", sockf, buf, (int)size);
if ( !sockf || !buf || !size || size==SIZE_MAX ) return 0;
zmq_msg_t msg;
err = zmq_msg_init_size (&msg, size);
if ( err != 0 ){
wrote = 0;
WRITE_FMT_LOG(LOG_ERR, "zmq_msg_init_size err %d, errno %d, status %s\n", err, zmq_errno(), zmq_strerror(zmq_errno()));
}
else{
memcpy (zmq_msg_data (&msg), buf, size);
WRITE_FMT_LOG(LOG_DEBUG, "zmq_sending fd=%d buf %d bytes...", sockf->fs_fd, (int)size );
err = zmq_send ( sockf->netw_socket, &msg, 0);
if ( err != 0 ){
WRITE_FMT_LOG(LOG_ERR, "zmq_send err %d, errno %d, status %s\n", err, zmq_errno(), zmq_strerror(zmq_errno()));
wrote = 0; /*nothing sent*/
}
else{
wrote = size;
WRITE_LOG(LOG_DEBUG, "zmq_send ok");
}
zmq_msg_close (&msg);
}
__bytes_sent +=wrote;
return wrote;
}
int zeromq_term(struct zeromq_pool* zpool){
int err = ERR_OK;
WRITE_FMT_LOG(LOG_DEBUG, "%p", zpool);
if ( !zpool ) return ERR_BAD_ARG;
/*free array*/
if ( zpool->sockf_array ){
for ( int i=0; i < zpool->count_max; i++ ){
free(zpool->sockf_array[i].tempbuf);
}
free(zpool->sockf_array), zpool->sockf_array = NULL;
}
/*destroy zmq context*/
if (zpool->context){
WRITE_LOG(LOG_ERR, "zmq_term trying...\n");fflush(0);
int err= zmq_term(zpool->context);
if ( err != 0 ){
err = ERR_ERROR;
WRITE_FMT_LOG(LOG_ERR, "zmq_term err %d, errno %d errtext %s\n", err, zmq_errno(), zmq_strerror(zmq_errno()));
}
else{
err= ERR_OK;
zpool->context = NULL;
WRITE_FMT_LOG(LOG_ERR, "zmq_term ok, sent %u ; recv %u bytes\n", __bytes_sent, __bytes_recv);
}
}
else{
WRITE_LOG(LOG_ERR, "context error NULL");
err = ERR_ERROR;
}
return err;
}
/*READ 1x struct packet_data_t {type EPACKET_SEQUENCE_REQUEST}
*READ 1x struct request_data_t
* @param dst_pid destination process should receive ranges*/
int
read_range_request( const char *readf, struct request_data_t* sequence ){
int len = 0;
int fdr = open(readf, O_RDONLY);
WRITE_FMT_LOG(LOG_DEBUG, "reading range request from %s (EPACKET_SEQUENCE_REQUEST)", readf);
int bytes;
struct packet_data_t t;
t.type = EPACKET_UNKNOWN;
bytes=read( fdr, (char*)&t, sizeof(t) );
WRITE_FMT_LOG(LOG_DEBUG, "r packet_data_t bytes=%d", bytes);
WRITE_FMT_LOG(LOG_DEBUG, "readed packet: type=%d, size=%d, src_node=%d", t.type, (int)t.size, t.src_nodeid );
if ( t.type == EPACKET_SEQUENCE_REQUEST )
{
struct request_data_t *data = NULL;
for ( int j=0; j < t.size; j++ ){
data = &sequence[j];
bytes=read(fdr, data, sizeof(struct request_data_t));
WRITE_FMT_LOG(LOG_DEBUG, "r packet_data_t bytes=%d", bytes);
WRITE_FMT_LOG(LOG_DEBUG, "recv range request %d %d %d", data->src_nodeid, data->first_item_index, data->last_item_index );
}
}
else{
WRITE_LOG(LOG_ERR, "channel_recv_sequences_request::packet Unknown");
assert(0);
}
close(fdr);
return len;
}
int test_sort_result( const BigArrayPtr unsorted, const BigArrayPtr sorted, int len ){
uint32_t unsorted_crc = 0;
uint32_t sorted_crc = 0;
uint32_t initial;
if ( len >=1 ){
initial = sorted[0];
unsorted_crc = (unsorted_crc+unsorted[0]% CRC_ATOM) % CRC_ATOM;
sorted_crc = (sorted_crc+sorted[0]% CRC_ATOM) % CRC_ATOM;
}
else return 1;
for ( int i=1; i < len; i++ ){
unsorted_crc = (unsorted_crc+unsorted[i]% CRC_ATOM) % CRC_ATOM;
sorted_crc = (sorted_crc+sorted[i]% CRC_ATOM) % CRC_ATOM;
if ( initial > sorted[i] ){
WRITE_FMT_LOG(LOG_UI, "initial %u > sorted[i] %u", initial, sorted[i]);
return 0;
}
else initial = sorted[i];
}
//crc test
if ( unsorted_crc != sorted_crc ){
WRITE_FMT_LOG(LOG_UI, "CRC_FAILED %u %u", unsorted_crc, sorted_crc);
return 0;
}
return 1;
}
int main(int argc, char *argv[]){
struct file_records_t file_records;
if ( argc > 2 ){
char logname[50];
sprintf(logname, "%s%s%s.log", CLIENT_LOG, DEST, argv[2]);
OPEN_LOG(logname, DEST, atoi(argv[2]));
file_records.vfs_path = argv[1];
int err = get_all_files_from_dbtable(DB_PATH, DEST, &file_records);
if ( err != 0 ) return 1;
}
else{
printf("usage: 1st arg: should be path to VFS folder, 2nd: unique node integer id\n");fflush(0);
return 1;
}
int nodeid = atoi(argv[2]);
WRITE_FMT_LOG(LOG_UI, "[%d] Destination node started", nodeid);
BigArrayPtr unsorted_array = NULL;
BigArrayPtr sorted_array = NULL;
unsorted_array = malloc( ARRAY_ITEMS_COUNT*sizeof(BigArrayItem) );
struct file_record_t* ranges_r_record = match_file_record_by_fd( &file_records, DEST_FD_READ_SORTED_RANGES_REP);
struct file_record_t* ranges_w_record = match_file_record_by_fd( &file_records, DEST_FD_WRITE_SORTED_RANGES_REP);
assert(ranges_r_record);
assert(ranges_w_record);
repreq_read_sorted_ranges( ranges_r_record->fpath, ranges_w_record->fpath, nodeid,
unsorted_array, ARRAY_ITEMS_COUNT, SRC_NODES_COUNT );
sorted_array = alloc_merge_sort( unsorted_array, ARRAY_ITEMS_COUNT );
/*save sorted array to output file*/
char outputfile[100];
memset(outputfile, '\0', 100);
sprintf(outputfile, DEST_FILE_FMT, nodeid );
int destfd = open(outputfile, O_WRONLY|O_CREAT);
if ( destfd >= 0 ){
const size_t data_size = sizeof(BigArrayItem)*ARRAY_ITEMS_COUNT;
if ( sorted_array ){
const ssize_t wrote = write(destfd, sorted_array, data_size);
assert(wrote == data_size );
}
close(destfd);
}
struct file_record_t* results_w_record = match_file_record_by_fd( &file_records, DEST_FD_WRITE_SORT_RESULT);
assert(results_w_record);
write_sort_result( results_w_record->fpath, nodeid, sorted_array, ARRAY_ITEMS_COUNT );
free(unsorted_array);
WRITE_FMT_LOG(LOG_UI, "[%d] Destination node complete", nodeid);
}
int close_all_comm_files(struct zeromq_pool* zpool){
WRITE_FMT_LOG(LOG_DEBUG, "%p", zpool);
int err = ERR_OK;
for (int i=0; i < zpool->count_max; i++){
if ( !zpool->sockf_array[i].unused )
err = close_sockf(zpool, sockf_by_fd(zpool, zpool->sockf_array[i].fs_fd));
if ( ERR_OK != err ){
WRITE_FMT_LOG(LOG_ERR, "fd=%d, error=%d", zpool->sockf_array[i].fs_fd, err);
return err;
}
}
return err;
}
/*i/o 2 files
*READ 1x struct request_data_t
*WRITE 1x char, reply
*READ 1x char is_complete flag, if 0
*WRITE 1x int, nodeid
*READ 1x char, reply
*WRITE 1x size_t, array_len
*READ 1x char, reply
*WRITE array_len x HistogramArrayItem, histograms array
* @return 1-should receive again, 0-complete request - it should not be listen again*/
int
read_requests_write_detailed_histograms( const char* readf, const char* writef, int nodeid,
const BigArrayPtr source_array, int array_len){
int fdr = open(readf, O_RDONLY);
int fdw = open(writef, O_WRONLY);
int is_complete = 0;
char reply;
do {
WRITE_FMT_LOG(LOG_DEBUG, "Reading from file %s detailed histograms request", readf );fflush(0);
/*receive data needed to create histogram using step=1,
actually requested histogram should contains array items range*/
struct request_data_t received_histogram_request;
int bytes;
bytes = read(fdr, (char*) &received_histogram_request, sizeof(received_histogram_request) );
assert(bytes>0);
WRITE_FMT_LOG(LOG_DEBUG, "r request_data_t bytes=%d", bytes);
bytes = write(fdw, &reply, 1);
WRITE_FMT_LOG(LOG_DEBUG, "w reply bytes=%d", bytes);
bytes = read(fdr, (char*) &is_complete, sizeof(is_complete) );
WRITE_FMT_LOG(LOG_DEBUG, "r is_complete bytes=%d", bytes);
int histogram_len = 0;
//set to our offset, check it
int offset = min(received_histogram_request.first_item_index, array_len-1 );
int requested_length = received_histogram_request.last_item_index - received_histogram_request.first_item_index;
requested_length = min( requested_length, array_len - offset );
HistogramArrayPtr histogram = alloc_histogram_array_get_len( source_array, offset, requested_length, 1, &histogram_len );
size_t sending_array_len = histogram_len;
/*Response to request, entire reply contains requested detailed histogram*/
bytes=write(fdw, &nodeid, sizeof(int) );
WRITE_FMT_LOG(LOG_DEBUG, "w node_id bytes=%d", bytes);
bytes=read(fdr, &reply, 1);
WRITE_FMT_LOG(LOG_DEBUG, "r reply bytes=%d", bytes);
bytes=write(fdw, &sending_array_len, sizeof(size_t) );
WRITE_FMT_LOG(LOG_DEBUG, "w sending_array_len bytes=%d", bytes);
bytes=read(fdr, &reply, 1);
WRITE_FMT_LOG(LOG_DEBUG, "r reply bytes=%d", bytes);
bytes=write(fdw, histogram, histogram_len*sizeof(HistogramArrayItem) );
WRITE_FMT_LOG(LOG_DEBUG, "r array histogram_len bytes=%d", bytes);
free( histogram );
WRITE_FMT_LOG(LOG_DETAILED_UI, "histograms wrote into file %s", writef );
}while(!is_complete);
close(fdr);
close(fdw);
return is_complete;
}
int open_all_comm_files(struct zeromq_pool* zpool, struct db_records_t *db_records){
WRITE_FMT_LOG(LOG_DEBUG, "%p, %p", zpool, db_records);
int err = ERR_OK;
for (int i=0; i < db_records->count; i++){
if ( EFILE_MSQ == db_records->array[i].ftype ){
struct db_record_t *record = &db_records->array[i];
if ( !open_sockf(zpool, db_records, record->fd) ){
WRITE_FMT_LOG(LOG_ERR, "fd=%d, error NULL sockf", db_records->array[i].fd);
return ERR_ERROR;
}
}
}
return err;
}
int add_sockf_copy_to_array(struct zeromq_pool* zpool, struct sock_file_t* sockf){
WRITE_FMT_LOG(LOG_DEBUG, "%p, %p", zpool, sockf);
int err = ERR_OK;
if ( !zpool || !sockf ) return ERR_BAD_ARG;
if ( !zpool->sockf_array) return ERR_BAD_ARG;
if( sockf_by_fd(zpool, sockf->fs_fd) ) return ERR_ALREADY_EXIST;
struct sock_file_t* sockf_add = NULL;
do{
/*search unused array cell*/
for (int i=0; i < zpool->count_max; i++)
if ( zpool->sockf_array[i].unused ){
sockf_add = &zpool->sockf_array[i];
break;
}
/*extend array if not found unused cell*/
if ( !sockf_add ){
zpool->count_max+=ESOCKF_ARRAY_GRANULARITY;
zpool->sockf_array = realloc(zpool->sockf_array, zpool->count_max * sizeof(struct sock_file_t));
if ( zpool->sockf_array ){
for (int i=zpool->count_max-ESOCKF_ARRAY_GRANULARITY; i < zpool->count_max; i++)
zpool->sockf_array[i].unused = 1;
}
else{
/*no memory re allocated
* This code section can't be covered by unit tests because it's system error*/
err = ERR_ERROR;
WRITE_LOG(LOG_ERR, "sockf_array realloc mem failed");
}
}
}while(!sockf_add || ERR_OK!=err);
*sockf_add = *sockf;
sockf_add->unused = 0;
return err;
}
struct sock_file_t* get_dual_sockf(struct zeromq_pool* zpool, struct db_records_t *db_records, int fd){
WRITE_FMT_LOG(LOG_DEBUG, "%p, %p", zpool, db_records);
/*search existing dual direction socket*/
struct sock_file_t *dual_sockf = NULL;
struct db_record_t* record1 = NULL;
struct db_record_t* record2 = NULL;
struct db_record_t* db_record = match_db_record_by_fd( db_records, fd);
if ( !db_record ) return NULL; /*no records found, can be illegal using */
/*Dual direction socket has identical socket params but fmode read / write mode is different*/
for ( int i=0; i < db_records->count; i++ ){
if ( !strcmp(db_records->array[i].endpoint, db_record->endpoint ) &&
db_records->array[i].method == db_record->method &&
db_records->array[i].sock == db_record->sock ){
if ( db_records->array[i].fmode == 'r' )
record1 = &db_records->array[i];
if ( db_records->array[i].fmode == 'w' )
record2 = &db_records->array[i];
}
}
/*If requested fd should be use dual direction socket*/
if ( record1 && record2 ){
struct sock_file_t *sockf1 = sockf_by_fd(zpool, record1->fd );
struct sock_file_t *sockf2 = sockf_by_fd(zpool, record2->fd );
/*if found socket then use exiting zeromq socket*/
if ( sockf1 )
dual_sockf = sockf1;
else if ( sockf2 )
dual_sockf = sockf2;
}
return dual_sockf;
}
/**@param global_array_index is used to save result to correct place*/
BigArrayPtr
alloc_merge(
const BigArrayPtr left_array, int left_array_len,
const BigArrayPtr right_array, int right_array_len ){
BigArrayPtr larray = left_array;
BigArrayPtr rarray = right_array;
BigArrayPtr result = malloc( sizeof(BigArrayItem) *(left_array_len+right_array_len));
if ( !result ){
WRITE_FMT_LOG(LOG_DEBUG, "NULL pointer, for len=%d\n", left_array_len+right_array_len );
return NULL;
}
int current_result_index = 0;
while ( left_array_len > 0 && right_array_len > 0 ){
if ( larray[0] <= rarray[0] ){
result[current_result_index++] = larray[0];
++larray;
--left_array_len;
}
else{
result[current_result_index++] = rarray[0];
++rarray;
--right_array_len;
}
}
//copy last item
if ( left_array_len > 0 ){
copy_array( result+current_result_index, larray, left_array_len );
}
if ( right_array_len > 0 ){
copy_array( result+current_result_index, rarray, right_array_len );
}
return result;
}
/*reading data
* 1x struct packet_data_t (packet type, size of array)
*writing data
* 1x char, reply
*reading data
* packet_data_t::size x BigArrayItem (sorted part of array)
*writing data
* 1x char, reply
* */
void
repreq_read_sorted_ranges( struct ChannelsConfigInterface *chan_if, BigArrayPtr dst_array, int dst_array_len )
{
int *src_nodes_list = NULL;
int src_nodes_count = chan_if->GetNodesListByType( chan_if, ESourceNode, &src_nodes_list );
#ifdef MERGE_ON_FLY
BigArrayPtr merge_array = calloc(sizeof(BigArrayItem), dst_array_len);
#endif
int recv_bytes_count = 0;
for (int i=0; i < src_nodes_count; i++)
{
struct UserChannel *channel = chan_if->Channel(chan_if, ESourceNode, src_nodes_list[i], EChannelModeRead);
WRITE_FMT_LOG(LOG_DEBUG, "repreq_read_sorted_ranges #%d\n", i);
struct packet_data_t t;
read_channel( channel->fd, (char*)&t, sizeof(t) );
if (EPACKET_RANGE != t.type ){
WRITE_FMT_LOG(LOG_DEBUG, "assert t.type=%d %d(EPACKET_RANGE)\n", t.type, EPACKET_RANGE);
assert(0);
}
#ifdef MERGE_ON_FLY
/* 1. Copy to merge array first chunk
* 2. Copy to merge array next chunk, starting from next item from first chunk
* 3. Merge both chunks and write merged result into dst_array
* */
int count_items_part1 = t.size/sizeof(BigArrayItem);
int count_items_part2 = recv_bytes_count/sizeof(BigArrayItem);
read_channel( channel->fd, (char*)&merge_array[count_items_part2], t.size );
/*if both chunks recevied*/
if ( count_items_part2 != 0 ){
/*merge both chunks*/
dst_array = merge( dst_array, merge_array, count_items_part2,
&merge_array[count_items_part2], count_items_part1 );
}
else{
strncpy((char*)dst_array, (const char*)merge_array, t.size );
}
#else
read_channel( channel->fd, (char*)&dst_array[recv_bytes_count/sizeof(BigArrayItem)], t.size );
#endif
recv_bytes_count += t.size;
}
WRITE_LOG(LOG_DEBUG, "channel_receive_sorted_ranges OK\n" );
}
/*writing data
*WRITE 1x uint32_t, crc*/
void
write_crc(const char *writef, uint32_t crc ){
int fdw = open(writef, O_WRONLY);
int bytes = 0;
bytes = write(fdw, &crc, sizeof(crc) );
WRITE_FMT_LOG(LOG_DEBUG, "w crc bytes=%d", bytes);
close(fdw);
}
/*i/o
*WRITE 1x struct packet_data_t [EPACKET_RANGE]
*WRITE array_len x BigArrayItem, array*/
void
write_sorted_ranges( int fdw, const struct request_data_t* sequence, const BigArrayPtr src_array ){
const int array_len = sequence->last_item_index - sequence->first_item_index + 1;
const BigArrayPtr array = src_array+sequence->first_item_index;
WRITE_FMT_LOG(LOG_DEBUG, "Sending array_len=%d; min=%u, max=%u\n", array_len, array[0], array[array_len-1]);
/*write header*/
struct packet_data_t t;
t.size = array_len*sizeof(BigArrayItem);
t.type = EPACKET_RANGE;
WRITE_FMT_LOG(LOG_DEBUG, "writing to %d: size=%d, type=%d (EPACKET_RANGE)\n", fdw, (int)t.size, t.type );
WRITE_FMT_LOG(LOG_DEBUG, "sizeof(struct packet_data_t)=%d\n", (int)sizeof(struct packet_data_t));
write_channel(fdw, (const char*) &t, sizeof(struct packet_data_t));
write_channel(fdw, (const char*) array, t.size);
write_channel(fdw, (const char*) "1", 1); //send at last small data
WRITE_LOG(LOG_DEBUG, "Reply from receiver OK;\n");
}
int zmain(int argc, char **argv){
/* argv[0] is node name
* expecting in format : "name-%d",
* format for single node without decimal id: "name" */
WRITE_FMT_LOG(LOG_DEBUG, "Source node started argv[0]=%s\n", argv[0] );
if ( argc < 2 ){
WRITE_LOG(LOG_ERR, "argv[1] is expected, items count need to be passed.\n" );
return -1;
}
set_items_count_to_sortjob( atoi(argv[1]) );
int ownnodeid= -1;
int extracted_name_len=0;
int res =0;
/*get node type names via environnment*/
char *source_node_type_text = getenv(ENV_SOURCE_NODE_NAME);
char *dest_node_type_text = getenv(ENV_DEST_NODE_NAME);
char *man_node_type_text = getenv(ENV_MAN_NODE_NAME);
assert(source_node_type_text);
assert(dest_node_type_text);
assert(man_node_type_text);
ownnodeid = ExtractNodeNameId( argv[0], &extracted_name_len );
/*nodename should be the same we got via environment and extracted from argv[0]*/
assert( !strncmp(source_node_type_text, argv[0], extracted_name_len ) );
if ( ownnodeid == -1 ) ownnodeid=1; /*node id not specified for single node by default assign nodeid=1*/
/*setup channels conf, now used static data but should be replaced by data from zrt*/
struct ChannelsConfigInterface chan_if;
SetupChannelsConfigInterface( &chan_if, ownnodeid, ESourceNode );
/***********************************************************************
Add channels configuration into config object */
res = AddAllChannelsRelatedToNodeTypeFromDir( &chan_if, IN_DIR, EChannelModeRead, EManagerNode, man_node_type_text );
assert( res == 0 );
res = AddAllChannelsRelatedToNodeTypeFromDir( &chan_if, IN_DIR, EChannelModeRead, EDestinationNode, dest_node_type_text );
assert( res == 0 );
res = AddAllChannelsRelatedToNodeTypeFromDir( &chan_if, OUT_DIR, EChannelModeWrite, EManagerNode, man_node_type_text );
assert( res == 0 );
res = AddAllChannelsRelatedToNodeTypeFromDir( &chan_if, OUT_DIR, EChannelModeWrite, EDestinationNode, dest_node_type_text );
assert( res == 0 );
/*add input channel into config*/
res = chan_if.AddChannel( &chan_if, EInputOutputNode, 1, STDIN, EChannelModeRead ) != NULL? 0: -1;
assert( res == 0 );
/*--------------*/
res = start_node(&chan_if, ownnodeid);
CloseChannels(&chan_if);
return res;
}
/*writing data:
* 1x struct sort_result*/
void
write_sort_result( int fdw, int nodeid, BigArrayPtr sorted_array, int len ){
if ( !len ) return;
uint32_t sorted_crc = array_crc( sorted_array, ARRAY_ITEMS_COUNT );
WRITE_FMT_LOG(LOG_DEBUG, "[%d] send_sort_result: min=%u, max=%u, crc=%u\n",
nodeid, sorted_array[0], sorted_array[len-1], sorted_crc );
struct sort_result result;
result.nodeid = nodeid;
result.min = sorted_array[0];
result.max = sorted_array[len-1];
result.crc = sorted_crc;
write_channel(fdw, (const char*)&result, sizeof(result));
}
int remove_sockf_from_array_by_fd(struct zeromq_pool* zpool, int fd){
WRITE_FMT_LOG(LOG_DEBUG, "%p", zpool);
if ( !zpool ) return ERR_BAD_ARG;
if ( !zpool->sockf_array ) return ERR_BAD_ARG;
int err = ERR_NOT_FOUND;
for (int i=0; i < zpool->count_max; i++)
if ( zpool->sockf_array[i].fs_fd == fd){
zpool->sockf_array[i].unused = 1;
err = ERR_OK;
break;
}
return err;
}
/*writing data
* 1x struct packet_data_t[EPACKET_HISTOGRAM]
* array_len x HistogramArrayItem*/
void
write_histogram( const char *writef, const struct Histogram *histogram ){
size_t array_size = sizeof(HistogramArrayItem)*(histogram->array_len);
struct packet_data_t t;
t.type = EPACKET_HISTOGRAM;
t.src_nodeid = histogram->src_nodeid;
t.size = array_size;
WRITE_FMT_LOG(LOG_DEBUG, "size=%d, type=%d, src_nodeid=%d", (int)t.size, t.type, t.src_nodeid);
int fdw = open(writef, O_WRONLY);
int bytes = 0;
size_t buf_size = sizeof(t)+array_size;
/*writing to pipeline whole packet*/
char *buffer = malloc(buf_size);
memset(buffer, '\0', buf_size );
memcpy(buffer, &t, sizeof(t));
bytes = sizeof(t);
memcpy(buffer+bytes, histogram->array, array_size);
bytes = write(fdw, buffer, buf_size);
WRITE_FMT_LOG(LOG_DEBUG, "w array bytes=%d", bytes);
close(fdw);
}
/*writing data
* 1x struct packet_data_t[EPACKET_HISTOGRAM]
* array_len x HistogramArrayItem*/
void
write_histogram( int fdw, const struct Histogram *histogram ){
size_t array_size = sizeof(HistogramArrayItem)*(histogram->array_len);
struct packet_data_t t;
t.type = EPACKET_HISTOGRAM;
t.src_nodeid = histogram->src_nodeid;
t.size = array_size;
WRITE_FMT_LOG(LOG_DEBUG, "write histogram: fd=%d, size=%d, type=%d, src_nodeid=%d\n", fdw,
(int)t.size, t.type, t.src_nodeid);
/*writing to pipeline whole packet*/
write_channel(fdw, (const char*)&t, sizeof(t));
write_channel(fdw, (const char*)histogram->array, array_size);
WRITE_LOG(LOG_DEBUG, "write histogram OK\n");
}
int main(int argc, char *argv[]){
struct file_records_t file_records;
int sourcefd = -1;
if ( argc > 2 ){
char logname[50];
sprintf(logname, "%s%s%s.log", CLIENT_LOG, SOURCE, argv[2]);
OPEN_LOG(logname, SOURCE, atoi(argv[2]));
file_records.vfs_path = argv[1];
int err = get_all_files_from_dbtable(DB_PATH, SOURCE, &file_records);
WRITE_FMT_LOG(LOG_ERR, "get_all_files_from_dbtable err=%d", err);
if ( err != 0 ) return 1;
}
else{
printf("usage: 1st arg: should be path to VFS folder, 2nd: unique node integer id\n");fflush(0);
return 1;
}
int nodeid = atoi(argv[2]);
BigArrayPtr unsorted_array = NULL;
BigArrayPtr partially_sorted_array = NULL;
/*get unsorted data*/
char inputfile[100];
memset(inputfile, '\0', 100);
sprintf(inputfile, SOURCE_FILE_FMT, nodeid );
sourcefd = open(inputfile, O_RDONLY);
if ( sourcefd >= 0 ){
const size_t data_size = sizeof(BigArrayItem)*ARRAY_ITEMS_COUNT;
unsorted_array = malloc( data_size );
if ( unsorted_array ){
const ssize_t readed = read(sourcefd, unsorted_array, data_size);
assert(readed == data_size );
}
close(sourcefd);
}
else{
WRITE_FMT_LOG(LOG_ERR, "Can not open input file %s", inputfile);
exit(0);
}
/*sort data locally*/
partially_sorted_array = alloc_merge_sort( unsorted_array, ARRAY_ITEMS_COUNT );
//if first part of sorting in single thread are completed
if ( test_sort_result( unsorted_array, partially_sorted_array, ARRAY_ITEMS_COUNT ) ){
if ( ARRAY_ITEMS_COUNT ){
WRITE_FMT_LOG(LOG_UI, "Single process sorting complete min=%u, max=%u: TEST OK.\n",
partially_sorted_array[0], partially_sorted_array[ARRAY_ITEMS_COUNT-1] );
}
/*send crc of sorted array to the manager node*/
uint32_t crc = array_crc( partially_sorted_array, ARRAY_ITEMS_COUNT );
WRITE_FMT_LOG(LOG_DEBUG, "crc=%u", crc);
struct file_record_t* write_crc_r = match_file_record_by_fd( &file_records, SOURCE_FD_WRITE_CRC);
WRITE_FMT_LOG(LOG_DEBUG, "SOURCE_FD_WRITE_CRC fd=%p", write_crc_r);
assert(write_crc_r);
write_crc( write_crc_r->fpath, crc );
WRITE_LOG(LOG_DEBUG, "crc wrote");
/*send of crc complete*/
int histogram_len = 0;
HistogramArrayPtr histogram_array = alloc_histogram_array_get_len(
partially_sorted_array, 0, ARRAY_ITEMS_COUNT, 1000, &histogram_len );
struct Histogram single_histogram;
single_histogram.src_nodeid = nodeid;
single_histogram.array_len = histogram_len;
single_histogram.array = histogram_array;
//send histogram to manager
struct file_record_t* write_hist_r = match_file_record_by_fd( &file_records, SOURCE_FD_WRITE_HISTOGRAM);
assert(write_hist_r);
write_histogram( write_hist_r->fpath, &single_histogram );
struct file_record_t* read_dhist_req_r = match_file_record_by_fd( &file_records, SOURCE_FD_READ_D_HISTOGRAM_REQ);
struct file_record_t* write_dhist_req_r = match_file_record_by_fd( &file_records, SOURCE_FD_WRITE_D_HISTOGRAM_REQ);
assert(read_dhist_req_r);
assert(write_dhist_req_r);
read_requests_write_detailed_histograms( read_dhist_req_r->fpath, write_dhist_req_r->fpath, nodeid,
partially_sorted_array, ARRAY_ITEMS_COUNT );
WRITE_LOG(LOG_DETAILED_UI, "\n!!!!!!!Histograms Sending complete!!!!!!.\n");
struct request_data_t req_data_array[SRC_NODES_COUNT];
init_request_data_array( req_data_array, SRC_NODES_COUNT);
//////////////////////////////
struct file_record_t* read_sequnce_req_r = match_file_record_by_fd( &file_records, SOURCE_FD_READ_SEQUENCES_REQ);
assert(read_sequnce_req_r);
read_range_request( read_sequnce_req_r->fpath, req_data_array );
//////////////////////////////
//////////////////////////////
for ( int i=0; i < SRC_NODES_COUNT; i++ ){
int dst_nodeid = req_data_array[i].dst_nodeid;
int dst_write_fd = dst_nodeid - FIRST_DEST_NODEID + SOURCE_FD_WRITE_SORTED_RANGES_REQ;
int dst_read_fd = dst_nodeid - FIRST_DEST_NODEID + SOURCE_FD_READ_SORTED_RANGES_REQ;
WRITE_FMT_LOG(LOG_DEBUG, "write_sorted_ranges fdw=%d, fdr=%d", dst_write_fd, dst_read_fd );
struct file_record_t* write_ranges_req_r = match_file_record_by_fd( &file_records, dst_write_fd);
struct file_record_t* read_ranges_req_r = match_file_record_by_fd( &file_records, dst_read_fd);
assert(write_ranges_req_r);
assert(read_ranges_req_r);
WRITE_FMT_LOG(LOG_DEBUG, "req_data_array[i].dst_nodeid=%d", req_data_array[i].dst_nodeid );
write_sorted_ranges( write_ranges_req_r->fpath, read_ranges_req_r->fpath,
&req_data_array[i], partially_sorted_array );
}
WRITE_FMT_LOG(LOG_DETAILED_UI, "[%d]Sending Ranges Complete-OK", nodeid);
//////////////////////////////
free(unsorted_array);
free(partially_sorted_array);
}
else{
WRITE_LOG(LOG_UI, "Single process sorting failed: TEST FAILED.\n");
exit(0);
}
}
ssize_t read_sockf(struct sock_file_t *sockf, char *buf, size_t count){
WRITE_FMT_LOG(LOG_DEBUG, "%p, %p, %d", sockf, buf, (int)count);
if ( !sockf || !buf || !count || count==SIZE_MAX ) return 0;
/*use sockf->tempbuf to save received results if user received more data than can be write into buf
* Data from temp buf should be readed at next call of read*/
int bytes_read = 0;
WRITE_FMT_LOG(LOG_DEBUG, "count=%d", (int)count);
if ( sockf->tempbuf && sockf->tempbuf->buf ){
WRITE_FMT_LOG(LOG_DEBUG, "read from tempbuf, size=%d, pos=%d", (int)sockf->tempbuf->size, sockf->tempbuf->pos );
/*how much of data can be copied, but not more than requested*/
bytes_read = min( sockf->tempbuf->size-sockf->tempbuf->pos, count );
/*copy to buf param from temp buffer starting from current pos*/
memcpy (buf, sockf->tempbuf->buf+sockf->tempbuf->pos, bytes_read);
/*update temp buf pos, increase for amount bytes read*/
sockf->tempbuf->pos += bytes_read;
/*temp buf completely readed*/
if ( sockf->tempbuf->size <= sockf->tempbuf->pos ){
free( sockf->tempbuf->buf ), sockf->tempbuf->buf = NULL;
free( sockf->tempbuf ), sockf->tempbuf = NULL;
if ( bytes_read < count ){
/*rest of data should be readed from zmq socket*/
count = count-bytes_read;
buf = buf+bytes_read; /*set new offset just after wrote data*/
}
else{
__bytes_recv += bytes_read;
return bytes_read;
}
}
else{
__bytes_recv += bytes_read;
return bytes_read;
}
}
int bytes = 0;
if ( sockf->netw_socket ){
do{
zmq_msg_t msg;
zmq_msg_init (&msg);
WRITE_FMT_LOG(LOG_DEBUG, "zmq_recv fd=%d, %d bytes...", sockf->fs_fd, (int)count);
int err = zmq_recv ( sockf->netw_socket, &msg, 0);
if ( 0 != err ){
/*read error*/
WRITE_FMT_LOG(LOG_DEBUG, "zmq_recv err %d, errno %d, status %s", err, zmq_errno(), zmq_strerror(zmq_errno()) );
return 0;
}
/*read ok*/
size_t msg_size = zmq_msg_size (&msg);
WRITE_FMT_LOG(LOG_ERR, "zmq_recv %d bytes ok\n", (int)msg_size);
bytes = min( msg_size, count );
void *recv_data = zmq_msg_data (&msg);
memcpy (buf, recv_data, bytes);
//if readed data less than received data, then store rest in tempbuf
if ( bytes < msg_size ){
sockf->tempbuf = malloc( sizeof(struct tempbuf_t) );
/*initialization of all tempbuf members*/
sockf->tempbuf->buf = malloc( msg_size-bytes );
memset(sockf->tempbuf->buf, '\0', msg_size-bytes);
memcpy( sockf->tempbuf->buf, recv_data+bytes, msg_size-bytes );
sockf->tempbuf->size = msg_size-bytes;
sockf->tempbuf->pos=0;
}
}while(bytes_read+bytes < count);
}
__bytes_recv += bytes_read+bytes;
return bytes_read+bytes;
}
struct sock_file_t* open_sockf(struct zeromq_pool* zpool, struct db_records_t *db_records, int fd){
WRITE_FMT_LOG(LOG_DEBUG, "%p, %p, %d", zpool, db_records, fd);
if (!zpool || !db_records) return NULL;
struct sock_file_t *sockf = sockf_by_fd(zpool, fd );
if ( sockf ) {
/*file with predefined descriptor already opened, just return socket*/
WRITE_FMT_LOG(LOG_DEBUG, "Existing socket: Trying to open twice? %d", sockf->fs_fd);
return sockf;
}
/* Trying to open new msq file because socket asociated with file descriptor is not found,
* trying to open socket in normal way, first search socket data associated with file descriptor
* in channels DB; From found db_record retrieve socket details data and start sockf opening flow;
* Flow1: For non existing, non dual socket add new socket record and next create&init zmq network socket;
* Flow2: For file descriptor with existing associated dual direction socket, add new socket record into
* array of sockets, but use already existing zmq networking socket for both 'r','w' descriptors;*/
struct db_record_t* db_record = match_db_record_by_fd( db_records, fd);
if ( db_record ){
/*create new socket record {sock_file_t}*/
sockf = malloc( sizeof(struct sock_file_t) );
if ( !sockf ){
/*no memory allocated
* This code section can't be covered by unit tests because it's not depends from input params*/
WRITE_LOG(LOG_ERR, "sockf malloc NULL");
}
else{
memset(sockf, '\0', sizeof(struct sock_file_t));
sockf->fs_fd = db_record->fd;
sockf->sock_type = db_record->sock;
sockf->tempbuf = NULL;
/*search existing dual direction socket should be only one zeromq socket for two files with
* the same endpoint (it's a trick for dual direction sockets)*/
struct sock_file_t* dual_sockf = get_dual_sockf(zpool, db_records, fd);
if ( dual_sockf ){
/*Flow2: dual direction socket, use existing zmq network socket*/
WRITE_LOG(LOG_DEBUG, "use dual socket");
sockf->netw_socket = dual_sockf->netw_socket;
}
else{
/*Flow1: init zmq network socket in normal way*/
WRITE_FMT_LOG(LOG_DEBUG, "open socket: %s, sock type %d\n", db_record->endpoint, db_record->sock);
sockf->netw_socket = zmq_socket( zpool->context, db_record->sock );
if ( !sockf->netw_socket ){
WRITE_LOG(LOG_ERR, "zmq_socket return NULL");
free(sockf), sockf=NULL;
}
else{
int err = ERR_OK;
switch(db_record->method){
case EMETHOD_BIND:
WRITE_LOG(LOG_DEBUG, "open socket: bind");
err= zmq_bind(sockf->netw_socket, db_record->endpoint);
WRITE_FMT_LOG(LOG_ERR, "zmq_bind status err %d, errno %d, status %s\n", err, zmq_errno(), zmq_strerror(zmq_errno()));
break;
case EMETHOD_CONNECT:
WRITE_LOG(LOG_DEBUG, "open socket: connect");
err = zmq_connect(sockf->netw_socket, db_record->endpoint);
WRITE_FMT_LOG(LOG_DEBUG, "zmq_connect status err %d, errno %d, status %s\n", err, zmq_errno(), zmq_strerror(zmq_errno()));
break;
default:
WRITE_LOG(LOG_ERR, "open socket: undefined sock method");
err = ERR_ERROR;
break;
}
if ( err != ERR_OK ){
WRITE_LOG(LOG_ERR, "close opened socket, free sockf, because connect|bind failed");
zmq_close( sockf->netw_socket );
free(sockf), sockf = NULL;
}
}
}
if ( sockf ){
int err = add_sockf_copy_to_array(zpool, sockf);
if ( ERR_OK != err ){
/* This code section can't be covered by unit tests because it's system relative error
* and additionaly function check ERR_ALREADY_EXIST case and return it*/
WRITE_FMT_LOG(LOG_ERR, "add_sockf_copy_to_array %d", err);
/*if socket opened and used by another sockf, then do not close it*/
if ( !dual_sockf ){
free(sockf->netw_socket);
free(sockf);
}
else{
zmq_close(sockf->netw_socket);
}
}
}
}
}
return sockf;
}
int start_node( struct ChannelsConfigInterface *chan_if, int nodeid ){
BigArrayPtr sorted_array = NULL;
const size_t data_size = sizeof(BigArrayItem)*ARRAY_ITEMS_COUNT;
/*If can use bitonic sort*/
if ( test_sse41_CPU() ){
WRITE_LOG(LOG_ERR, "allocate bitonic sort memories\n");
/*we needed an extra buf to use with bitonic sort*/
BigArrayPtr extra_buf = NULL;
/*allocated memory should be aligned*/
extra_buf = aligned_malloc(data_size, SSE2_ALIGNMENT);
if ( !extra_buf ) {
WRITE_LOG(LOG_ERR, "Can't allocate memories\n");
abort();
}
sorted_array = aligned_malloc(data_size, SSE2_ALIGNMENT);
if (!sorted_array) {
WRITE_LOG(LOG_ERR, "Can't allocate memories\n");
abort();
}
/*Read source data from STDIN*/
struct UserChannel *channel = chan_if->Channel(chan_if, EInputOutputNode, 1, EChannelModeRead );
const ssize_t readed = read( channel->fd, (void*)sorted_array, data_size);
WRITE_FMT_LOG(LOG_ERR, "readed input file, expected size=%d, read size=%d\n", data_size, readed);
assert(readed == data_size );
WRITE_LOG(LOG_DETAILED_UI, "Start bitonic sorting\n");
bitonic_sort_chunked((float*)sorted_array, ARRAY_ITEMS_COUNT, (float*)extra_buf, DEFAULT_CHUNK_SIZE);
WRITE_LOG(LOG_DETAILED_UI, "Bitonic sorting complete\n");
aligned_free(extra_buf);
}
/*If can't use bitonic - then use c qsort*/
else{
WRITE_LOG(LOG_ERR, "qsort will used\n");
BigArrayPtr unsorted_array = NULL;
unsorted_array = malloc( data_size );
if ( !unsorted_array ) {
WRITE_LOG(LOG_ERR, "Can't allocate memories\n");
abort();
}
if ( unsorted_array ){
/*Read source data from STDIN*/
const ssize_t readed = read( STDIN, (void*)unsorted_array, data_size);
WRITE_FMT_LOG(LOG_ERR, "readed input file, expected size=%d, read size=%d\n", data_size, readed);
assert(readed == data_size );
}
WRITE_LOG(LOG_DETAILED_UI, "Start qsort sorting\n");
sorted_array = alloc_copy_array( unsorted_array, ARRAY_ITEMS_COUNT );
qsort( sorted_array, ARRAY_ITEMS_COUNT, sizeof(BigArrayItem), quicksort_BigArrayItem_comparator );
free(unsorted_array);
}
struct UserChannel *channel = chan_if->Channel( chan_if, EManagerNode, 1, EChannelModeWrite );
assert(channel);
#if LOG_LEVEL == LOG_DEBUG
channel->DebugPrint(channel, stderr);
#endif
/*send crc of sorted array to the manager node*/
uint32_t crc = array_crc( sorted_array, ARRAY_ITEMS_COUNT );
WRITE_FMT_LOG(LOG_DEBUG, "write crc=%u into fd=%d", crc, channel->fd);
write_crc( channel->fd, crc );
/*send of crc complete*/
/*prepare histogram data, with step defined by BASE_HISTOGRAM_STEP*/
int histogram_len = 0;
HistogramArrayPtr histogram_array = alloc_histogram_array_get_len(
sorted_array, 0, ARRAY_ITEMS_COUNT, BASE_HISTOGRAM_STEP, &histogram_len );
WRITE_LOG(LOG_DEBUG, "histogram prepared, sending...");
struct Histogram single_histogram;
single_histogram.src_nodeid = nodeid;
single_histogram.array_len = histogram_len;
single_histogram.array = histogram_array;
/*send histogram to manager*/
channel = chan_if->Channel(chan_if, EManagerNode, 1, EChannelModeWrite);
assert(channel);
write_histogram( channel->fd, &single_histogram );
struct UserChannel *chanw = chan_if->Channel(chan_if, EManagerNode, 1, EChannelModeWrite);
channel = chan_if->Channel(chan_if, EManagerNode, 1, EChannelModeRead);
assert(channel);
assert(chanw);
/*read request for detailed histogram and send it to manager*/
read_requests_write_detailed_histograms( channel->fd, chanw->fd, nodeid, sorted_array, ARRAY_ITEMS_COUNT );
WRITE_LOG(LOG_UI, "\n!!!!!!!Histograms Sending complete!!!!!!.\n");
/* source nodes count not available because not has channels intended to communicate with source nodes
* therefore will use dest nodes count because it's equal to source nodes count */
int *dst_nodes_list = NULL;
int dst_nodes_count = chan_if->GetNodesListByType(chan_if, EDestinationNode, &dst_nodes_list );
int src_nodes_count = dst_nodes_count;
WRITE_FMT_LOG( LOG_DEBUG, "src_nodes_count=%d\n", src_nodes_count );
/*read range request (data start, end, dest node id) from manager node*/
struct request_data_t req_data_array[src_nodes_count];
init_request_data_array( req_data_array, src_nodes_count);
channel = chan_if->Channel( chan_if, EManagerNode, 1, EChannelModeRead );
assert(channel);
read_range_request( channel->fd, req_data_array );
WRITE_FMT_LOG( LOG_UI, "qsort array len=%d\n", src_nodes_count);
/*sort request data by dest node id to be deterministic*/
qsort( req_data_array, src_nodes_count, sizeof(struct request_data_t), quicksort_reqdata_by_destnodeid_comparator );
WRITE_LOG( LOG_UI, "qsort OK" );
/*send array data to the destination nodes, bounds for pieces of data was
* received previously with range request */
for ( int i=0; i < src_nodes_count; i++ ){
int dst_nodeid = req_data_array[i].dst_nodeid;
channel = chan_if->Channel( chan_if, EDestinationNode, dst_nodeid, EChannelModeWrite );
int dst_write_fd = channel->fd;
WRITE_FMT_LOG(LOG_DEBUG, "write_sorted_ranges write fd=%d", dst_write_fd );
WRITE_FMT_LOG(LOG_DEBUG, "req_data_array[i].dst_nodeid=%d", req_data_array[i].dst_nodeid );
write_sorted_ranges( dst_write_fd, &req_data_array[i], sorted_array );
}
WRITE_LOG(LOG_UI, "Sending Ranges Complete-OK");
if ( test_sse41_CPU() )
aligned_free(sorted_array);
else
free(sorted_array);
return 0;
}
/*i/o 2 files
*WRITE 1x struct packet_data_t [EPACKET_RANGE]
*READ 1x char, reply
*WRITE array_len x BigArrayItem, array
*READ 1x char, reply*/
void
write_sorted_ranges( const char *writef, const char *readf,
const struct request_data_t* sequence, const BigArrayPtr src_array ){
int fdw = open(writef, O_WRONLY);
int fdr = open(readf, O_RDONLY);
const int array_len = sequence->last_item_index - sequence->first_item_index + 1;
const BigArrayPtr array = src_array+sequence->first_item_index;
WRITE_FMT_LOG(LOG_DEBUG, "Sending array_len=%d; min=%d, max=%d", array_len, array[0], array[array_len-1]);
int bytes;
char unused_reply;
#ifdef FUSE_CLIENT
/*FUSE server does not support data size > 128KB passed trying to write into file
* therefore divide array data into smaller blocks FUSE_IO_MAX_CHUNK_SIZE*/
int length_of_array_part = FUSE_IO_MAX_CHUNK_SIZE / sizeof(BigArrayItem);
/*get size of array part alligned by item size*/
int size_of_array_part = length_of_array_part * sizeof(BigArrayItem);
int sent_array_size = 0;
/*write into header complete size of sending array, size field contain sending array size*/
struct packet_data_t range_header;
range_header.size = array_len*sizeof(BigArrayItem);
range_header.type = EPACKET_RANGE;
bytes=write(fdw, (char*) &range_header, sizeof(struct packet_data_t) );
WRITE_FMT_LOG(LOG_DEBUG, "w header EPACKET_RANGE bytes=%d", bytes);
bytes=read( fdr, &unused_reply, 1 );
WRITE_FMT_LOG(LOG_DEBUG, "r reply bytes=%d", bytes);
struct packet_data_t range_part_header;
range_part_header.type = EPACKET_RANGE_PART;
while( sent_array_size < range_header.size ){
int sending_data_size = min(size_of_array_part, range_header.size - sent_array_size );
range_part_header.size = sending_data_size;
int buf_size = sizeof(struct packet_data_t) + sending_data_size;
char *buffer = malloc(buf_size);
/*write header into buffer*/
memcpy(buffer, &range_part_header, sizeof(struct packet_data_t) );
WRITE_FMT_LOG(LOG_DEBUG, "w into buffer EPACKET_RANGE_PART bytes=%d", (int)sizeof(struct packet_data_t));
/*write array data into buffer*/
int sent_array_items = sent_array_size / sizeof(BigArrayItem);
memcpy(buffer+sizeof(struct packet_data_t), array+sent_array_items, sending_data_size );
WRITE_FMT_LOG(LOG_DEBUG, "w into buffer array bytes=%d", sending_data_size);
/*write buffer into file*/
bytes=write(fdw, buffer, buf_size);
WRITE_FMT_LOG(LOG_DEBUG, "w array bytes=%d", bytes);
bytes=read( fdr, &unused_reply, 1 );
WRITE_FMT_LOG(LOG_DEBUG, "r reply bytes=%d", bytes);
free(buffer);
sent_array_size += sending_data_size;
}
#else
/*write header*/
struct packet_data_t t;
t.size = array_len*sizeof(BigArrayItem);
t.type = EPACKET_RANGE;
WRITE_FMT_LOG(LOG_DEBUG, "writing to %s: size=%d, type=%d (EPACKET_RANGE)", writef, (int)t.size, t.type );
WRITE_FMT_LOG(LOG_DEBUG, "sizeof(struct packet_data_t)=%d", (int)sizeof(struct packet_data_t));
bytes=write(fdw, (char*) &t, sizeof(struct packet_data_t) );
WRITE_FMT_LOG(LOG_DEBUG, "w packet_data_t bytes=%d", bytes);
bytes=read( fdr, &unused_reply, 1 );
WRITE_FMT_LOG(LOG_DEBUG, "r reply bytes=%d", bytes);
/*write array data*/
bytes=write(fdw, array, t.size);
WRITE_FMT_LOG(LOG_DEBUG, "w array bytes=%d", bytes);
WRITE_FMT_LOG(LOG_DEBUG, "Reading from %s receiver reply;", readf);
bytes=read( fdr, &unused_reply, 1 );
WRITE_FMT_LOG(LOG_DEBUG, "r reply bytes=%d", bytes);
#endif
WRITE_LOG(LOG_DEBUG, "Reply from receiver OK;");
close(fdw);
close(fdr);
}
