static INT64_T do_matrix_list(int argc, char **argv)
{
char path[CHIRP_PATH_MAX];
struct chirp_matrix *m;
complete_remote_path(argv[1], path);
m = chirp_matrix_open(current_host, path, stoptime);
if(m) {
printf("host: %s\n", current_host);
printf("path: %s\n", path);
printf("width: %d\n", chirp_matrix_width(m));
printf("height: %d\n", chirp_matrix_height(m));
printf("esize: %d\n", chirp_matrix_element_size(m));
printf("nhosts: %d\n", chirp_matrix_nhosts(m));
printf("nfiles: %d\n", chirp_matrix_nfiles(m));
chirp_matrix_close(m, stoptime);
return 0;
} else {
return 1;
}
}
int main(int argc, char *argv[])
{
int did_explicit_auth = 0;
int follow_mode = 0;
time_t stoptime;
signed char c;
int setAindex, setBindex; // funcindex;
FILE *setA = NULL;
FILE *setB = NULL;
char setApath[CHIRP_PATH_MAX];
char setBpath[CHIRP_PATH_MAX];
char *LIST_FILE_NAME = "set.list";
char setAfilename[CHIRP_PATH_MAX];
char setBfilename[CHIRP_PATH_MAX];
int len = CHIRP_PATH_MAX;
struct chirp_matrix *mat = NULL;
int mathost, matpath;
double *resbuff = NULL;
int numels;
int cntr;
// Variables defined by Li
int i; // for multiprocess calculation
int numOfMovingElements, numOfStableElements;
//int setACount, setBCount;
int setAPos, setBPos;
long setAStartPos;
//long setBStartPos;
int x1, y1, x2, y2, topLeftX, topLeftY; // [x1,y1]-start position, [x2,y2]-end position, the sub matrix we compute in a round
int x_rel, y_rel;
double threshold;
double threshold_min = 0;
double threshold_max = 1;
double threshold_interval = 0.2;
int count_thresholds;
int count_genuine = 0;
int count_impostar = 0;
//int count_fa = 0;
//int count_fr = 0;
double (*roc_data)[3];
int subject_equal;
// ~Variables defined by Li
int w, h, e, n;
w = 10;
h = 10;
e = 8;
n = 1;
x1 = y1 = x2 = y2 = -1;
topLeftX = topLeftY = 0;
debug_config(argv[0]);
while((c = getopt(argc, argv, "a:b:d:ft:vhw:i:e:n:x:y:p:q:r:s:X:Y:c:")) > -1) {
switch (c) {
case 'a':
auth_register_byname(optarg);
did_explicit_auth = 1;
break;
case 'b':
buffer_size = atoi(optarg);
break;
case 'd':
debug_flags_set(optarg);
break;
case 'f':
follow_mode = 1;
break;
case 't':
timeout = string_time_parse(optarg);
break;
case 'w':
w = atoi(optarg);
break;
case 'i':
h = atoi(optarg);
break;
case 'e':
e = atoi(optarg);
break;
case 'n':
n = atoi(optarg);
break;
case 'v':
cctools_version_print(stdout, argv[0]);
exit(0);
break;
case 'h':
show_help(argv[0]);
exit(0);
break;
case 'x':
numOfStableElements = atoi(optarg);
break;
case 'y':
numOfMovingElements = atoi(optarg);
break;
case 'p':
x1 = atoi(optarg);
break;
case 'q':
y1 = atoi(optarg);
break;
case 'r':
x2 = atoi(optarg);
break;
case 's':
y2 = atoi(optarg);
break;
case 'X':
topLeftX = atoi(optarg);
break;
case 'Y':
topLeftY = atoi(optarg);
break;
case 'c':
//numOfCores = atoi(optarg);
break;
}
}
cctools_version_debug(D_DEBUG, argv[0]);
if(!did_explicit_auth)
auth_register_all();
if((argc - optind) < 4) {
fprintf(stderr, "after all options, you must have: setA setB function mathost matpath\n");
exit(0);
}
stoptime = time(0) + timeout;
setAindex = optind;
setBindex = optind + 1;
mathost = optind + 2;
matpath = optind + 3;
// Set threshhold min, max and interval
threshold_min = 0;
threshold_max = 1;
threshold_interval = 0.01;
// Initialize result array - roc_data
count_thresholds = (threshold_max - threshold_min) / threshold_interval + 1;
roc_data = malloc(count_thresholds * 3 * sizeof(double));
if(!roc_data) {
fprintf(stderr, "Cannot initialize result buffer!\n");
exit(1);
}
for(i = 0, threshold = threshold_min; i < count_thresholds; i++, threshold += threshold_interval) {
roc_data[i][0] = threshold;
}
// Load matrix to be verified
printf("X1,X2,Y1,Y2: %i,%i,%i,%i\n", x1, x2, y1, y2);
mat = chirp_matrix_open(argv[mathost], argv[matpath], stoptime);
if(mat == NULL) {
fprintf(stderr, "No such matrix. Fail.\n");
exit(1);
}
printf("Start loading matrix ... \n");
printf("Width, height: %d, %d\n\n", chirp_matrix_width(mat), chirp_matrix_height(mat));
numels = (x2 - x1 + 1) * (y2 - y1 + 1);
resbuff = (double *) malloc(numels * sizeof(double));
double *pilot_resbuff;
pilot_resbuff = resbuff;
// TODO get_range function can get at most 10*10 matrix a time (actually it can get more)
/**
for(j=0; j<y2-y1+1; j++) {
for(i=0; i<x2-x1+1; i++) {
int matrtn = chirp_matrix_get_range( mat, x1+i, y1+j, 1, 1, pilot_resbuff, stoptime); //(x2-x1+1), (y2-y1+1), resbuff, stoptime );
printf("%.2f\t", (*pilot_resbuff));
if(matrtn == -1) printf("return mat error @ [%d, %d]!!\n", x1+i, y1+j);
pilot_resbuff++;
}
printf("\n");
}*/
int matrtn = chirp_matrix_get_range(mat, x1, y1, x2 - x1 + 1, y2 - y1 + 1, resbuff, stoptime); //(x2-x1+1), (y2-y1+1), resbuff, stoptime );
if(matrtn == -1) {
fprintf(stderr, "return mat error @ [%d, %d], width: %d; height: %d!\n", x1, y1, x2 - x1 + 1, y2 - y1 + 1);
exit(1);
}
printf("*******end of loading matrix********\n\n");
// Get local path for data sets directories
if(get_local_path(setApath, argv[setAindex], stoptime) != 0 || get_local_path(setBpath, argv[setBindex], stoptime) != 0) {
fprintf(stderr, "Paths to data sets are invalid!\n");
exit(1);
}
// setA and setB each contains a list of file names that points to the data files
char setAlistfile[CHIRP_PATH_MAX];
char setBlistfile[CHIRP_PATH_MAX];
strcpy(setAlistfile, setApath);
strcat(setAlistfile, LIST_FILE_NAME);
if((setA = fopen(setAlistfile, "r")) == NULL) {
fprintf(stderr, "Cannot open data set A list file - %s!\n", setAlistfile);
exit(1);
}
strcpy(setBlistfile, setBpath);
strcat(setBlistfile, LIST_FILE_NAME);
if((setB = fopen(setBlistfile, "r")) == NULL) {
fprintf(stderr, "Cannot open data set B list file - %s!\n", setBlistfile);
exit(1);
}
// Initialize position parameters and allocate memory for storing results of a block (sub matrix)
x_rel = y_rel = 0; // relative to the sub-matrix we are actually working on
// Go forward until line x1 in Set A list file
for(i = 0; i < x1 && !feof(setA); i++) {
fgets(setAfilename, len, setA);
}
if(i < x1) {
fprintf(stderr, "Set A has less then x1 elements!\n");
exit(1);
}
setAStartPos = ftell(setA);
// Go forward until line y1 in Set B list file
for(i = 0; i < y1 && !feof(setB); i++) {
fgets(setBfilename, len, setB);
}
if(i < y1) {
fprintf(stderr, "Set B has less then x1 elements!\n");
exit(1);
}
//setBStartPos = ftell(setB);
debug(D_CHIRP, "Matrix data:\n");
// start loop
fgets(setBfilename, len, setB);
if(setBfilename != NULL) {
size_t last = strlen(setBfilename) - 1;
if(setBfilename[last] == '\n')
setBfilename[last] = '\0';
}
for(setBPos = y1; !feof(setB) && setBPos <= y2; setBPos++) { // Set B - column of matrix
// Go directly to line y1 in Set B list file
fseek(setA, setAStartPos, SEEK_SET);
fgets(setAfilename, len, setA);
if(setAfilename != NULL) {
size_t last = strlen(setAfilename) - 1;
if(setAfilename[last] == '\n')
setAfilename[last] = '\0';
}
setAPos = x1;
for(setAPos = x1; !feof(setA) && setAPos <= x2; setAPos++) { // Set A- row of matrix
// Threshhold comparison
cntr = ((setBPos - y1) * (x2 - x1 + 1)) + (setAPos - x1);
subject_equal = isSubjectIdEqual(setAfilename, setBfilename);
if(subject_equal == 1) { // A genuine match
for(threshold = threshold_max, i = count_thresholds - 1; 1 - resbuff[cntr] < threshold; threshold -= threshold_interval, i--) {
// False reject
roc_data[i][1] += 1;
}
count_genuine++;
} else if(subject_equal == 0) { // A impostar match
for(threshold = threshold_min, i = 0; 1 - resbuff[cntr] >= threshold; threshold += threshold_interval, i++) {
// False accept
roc_data[i][2] += 1;
}
count_impostar++;
} else {
fprintf(stderr, "Cannot resolve filename in either %s or %s!\n", setAfilename, setBfilename);
exit(1);
}
debug(D_CHIRP, "%.2f\t", resbuff[cntr]);
fgets(setAfilename, len, setA);
if(setAfilename != NULL) {
size_t last = strlen(setAfilename) - 1;
if(setAfilename[last] == '\n')
setAfilename[last] = '\0';
}
}
debug(D_CHIRP, "\n");
fgets(setBfilename, len, setB);
if(setBfilename != NULL) {
size_t last = strlen(setBfilename) - 1;
if(setBfilename[last] == '\n')
setBfilename[last] = '\0';
}
}
printf("\n**********************************************************************\n");
// Printf roc_data
debug(D_CHIRP, "ROC raw data format: Threshold | False reject count | False accept count\n");
for(i = 0; i < count_thresholds; i++) {
debug(D_CHIRP, "%.2f\t%.2f\t%.2f;\t", roc_data[i][0], roc_data[i][1], roc_data[i][2]);
}
debug(D_CHIRP, "\n");
// Transform roc_data to ROC curve data
for(i = 0; i < count_thresholds; i++) {
roc_data[i][1] = 1 - (roc_data[i][1] / count_genuine); // 1 - FRR
roc_data[i][2] = roc_data[i][2] / count_impostar; // FAR
}
// Printf roc_data
debug(D_CHIRP, "ROC curve data format: Threshold | 1 - False reject rate | False accept rate\n");
for(i = 0; i < count_thresholds; i++) {
debug(D_CHIRP, "%.2f\t%.2f\t%.2f;\t", roc_data[i][0], roc_data[i][1], roc_data[i][2]);
}
debug(D_CHIRP, "\n");
// Write to dat file for gnuplot's use
FILE *roc_data_fp;
//char roc_line[20];
roc_data_fp = fopen("roc.dat", "w");
for(i = 0; i < count_thresholds; i++) {
fprintf(roc_data_fp, "%.2f\t%.2f\n", roc_data[i][1], roc_data[i][2]); // 1 - FRR, FAR
}
fclose(roc_data_fp);
free(resbuff);
debug(D_CHIRP, "%d comparisons in the matrix are tested! Genuine matches: %d\t Impostar matches: %d\n\n", cntr + 1, count_genuine, count_impostar);
printf("\nROC curve data generation completed successfully!\n%d comparisons in the matrix are tested!\n", cntr + 1);
return 0;
}
int main(int argc, char** argv) {
signed char cl;
int did_explicit_auth = 0;
int download,rm_local,rm_remote,rm_mat,file_provided;
int rm_remote_error = 0;
char matrix_target[CHIRP_PATH_MAX];
char finalize_file[CHIRP_PATH_MAX];
time_t stoptime = time(0)+3600;
download=rm_local=rm_remote=rm_mat=file_provided=0;
while((cl=getopt(argc,argv,"+a:d:hD:LRMF:")) > -1) {
switch(cl) {
case 'a':
auth_register_byname(optarg);
did_explicit_auth = 1;
break;
case 'd':
debug_flags_set(optarg);
break;
case 'h':
printUsage(argv[0]);
exit(0);
break;
case 'D': // download matrix data to local disk
download=1;
strcpy(matrix_target,optarg);
break;
case 'L': // force LOCAL state removal
rm_local=1;
break;
case 'R': // force REMOTE state removal (chirp_distribute -X)
rm_remote=1;
break;
case 'M': // force REMOTE MATRIX state removal
rm_mat=1;
break;
case 'F':
file_provided=1;
strcpy(finalize_file,optarg);
break;
}
}
if(!file_provided) {
fprintf(stderr,"Please provide argument -F [finalize file]\n");
printUsage(argv[0]);
exit(1);
}
if(!did_explicit_auth) auth_register_all(); // if an authentication mechanism wasn't chosen, default register all.
debug_config(argv[0]); // indicate what string to use as the executable name when printing debugging information
// first, parse finalize file to get information.
char* cmd;
char* wID;
char* local_dir;
char* mat_host;
char* mat_path;
char* remote_dir;
char* node_list;
char* hn;
char* fun_path;
int strlentmp;
FILE* fp = fopen(finalize_file,"r");
if(!fp) {
fprintf(stderr,"Finalize file not readable.\n");
exit(1);
}
// 0th item is workload id
if(fscanf(fp, " wID=%i ",&strlentmp) == 1) {
wID = malloc((strlentmp+1)*sizeof(char));
if(!wID)
{
fprintf(stderr,"Could not allocate %i bytes for workload ID\n",strlentmp);
exit(1);
}
if(fscanf(fp, " %s ", wID) != 1) {
fprintf(stderr,"Could not read in workload ID\n");
exit(2);
}
}
// first item is local prefix -- remove everything.
if(fscanf(fp, " local_dir=%i ",&strlentmp) == 1) {
local_dir = (char*) malloc((strlentmp+1)*sizeof(char));
if(!local_dir)
{
fprintf(stderr,"Could not allocate %i bytes for local directory\n",strlentmp);
exit(1);
}
if(fscanf(fp," %s ", local_dir) != 1) {
fprintf(stderr,"Could not read in local directory\n");
exit(2);
}
}
// second item is matrix host -\ remove
// third item is matrix path -/ matrix
if(fscanf(fp, " mat_host=%i ",&strlentmp) == 1) {
mat_host = (char *) malloc((strlentmp+1)*sizeof(char));
if(!mat_host)
{
fprintf(stderr,"Could not allocate %i bytes for matrix host\n",strlentmp);
exit(1);
}
if(fscanf(fp," %s ", mat_host) != 1) {
fprintf(stderr,"Could not read in matrix host\n");
exit(2);
}
}
if(fscanf(fp, " mat_path=%i ",&strlentmp) == 1) {
mat_path = (char *) malloc((strlentmp+1)*sizeof(char));
if(!mat_path)
{
fprintf(stderr,"Could not allocate %i bytes for matrix path\n",strlentmp);
exit(1);
}
if(fscanf(fp," %s ", mat_path) != 1) {
fprintf(stderr,"Could not read in matrix path\n");
exit(2);
}
}
// 4th item is chirp_dirname
if(fscanf(fp, " remote_dir=%i ",&strlentmp) == 1) {
remote_dir = (char *) malloc((strlentmp+1)*sizeof(char));
if(!remote_dir)
{
fprintf(stderr,"Could not allocate %i bytes for remote path\n",strlentmp);
exit(1);
}
if(fscanf(fp," %s ", remote_dir) != 1) {
fprintf(stderr,"Could not read in remote path\n");
exit(2);
}
}
if(rm_remote==1) {
fprintf(stderr,"Asked to remove remote state, but there is no remote state specified.\n");
rm_remote_error = 1;
}
// 7th item is full goodstring
if(fscanf(fp, " node_list=%i ",&strlentmp) == 1) {
node_list = (char *) malloc((strlentmp+1)*sizeof(char));
if(!node_list)
{
fprintf(stderr,"Could not allocate %i bytes for remote hosts\n",strlentmp);
exit(1);
}
if(fread(node_list,1,strlentmp,fp) != strlentmp) {
fprintf(stderr,"Could not read in remote hosts\n");
exit(2);
}
}
if(rm_remote==1 && rm_remote_error == 0) {
fprintf(stderr,"Asked to remove remote state, but there is no remote state specified.\n");
rm_remote_error = 1;
}
// 9th item is hostname
if(fscanf(fp, " host=%i ",&strlentmp) == 1) {
hn = malloc((strlentmp+1)*sizeof(char));
if(!hn)
{
fprintf(stderr,"Could not allocate %i bytes for hostname\n",strlentmp);
exit(1);
}
if(fscanf(fp, " %s ", hn) != 1) {
fprintf(stderr,"Could not read in hostname\n");
exit(2);
}
}
if(rm_remote==1 && rm_remote_error == 0) {
fprintf(stderr,"Asked to remove remote state, but there is no remote state specified.\n");
rm_remote_error = 1;
}
// 10th item is full function directory -- remove tarball, exception list
if(fscanf(fp, " fun_path=%i ",&strlentmp) == 1) {
fun_path = (char *) malloc((strlentmp+1)*sizeof(char));
if(!fun_path)
{
fprintf(stderr,"Could not allocate %i bytes for function directory\n",strlentmp);
exit(1);
}
if(fscanf(fp," %s ", fun_path) != 1) {
fprintf(stderr,"Could not read in function directory\n");
exit(2);
}
cmd = (char *) malloc((strlen(fun_path)+strlen("rm -f ")+strlen(wID)+strlen(".func.tar")+12)*sizeof(char));
if(!cmd)
{
fprintf(stderr,"Could not allocate memory for command\n");
exit(1);
}
sprintf(cmd,"rm -f %s/%s.func.tar",fun_path,wID);
if(system(cmd)) { fprintf(stderr,"Could not remove %s/%s.func.tar\n",fun_path,wID); exit(1);}
sprintf(cmd,"rm -f %s/exclude.list",fun_path);
if(system(cmd)) { fprintf(stderr,"Could not remove %s/exclude.list\n",fun_path); exit(1);}
free(cmd);
cmd=NULL;
}
else {
// internal function
fun_path = NULL;
}
// end parsing finalize file
// next, download if desired.
if(download) {
fprintf(stderr,"Download Matrix Mode\n");
FILE* mt = fopen(matrix_target, "w");
struct chirp_matrix* m = chirp_matrix_open( mat_host, mat_path, stoptime);
if(m) {
int w = chirp_matrix_width( m );
int h = chirp_matrix_height( m );
//int e = chirp_matrix_element_size( m );
double* buf = malloc(w*sizeof(double));
int x,y;
for(y=0; y < h; y++) {
chirp_matrix_get_row( m , y, buf, stoptime );
for(x=0; x<w; x++) {
fprintf(mt,"%i %i %.2lf\n",y,x,buf[x]);
}
}
}
else
{
printf("Could not open matrix %s %s\n",mat_host,mat_path);
return 1;
}
}
// next, delete remote state if desired.
if(rm_remote && !rm_remote_error) {
fprintf(stderr,"Remove Remote State Mode\n");
cmd = (char *) malloc(strlen("chirp_distribute -a hostname -X ")+10+(2*strlen(hn))+1+strlen(remote_dir)+1+strlen(node_list)+1);
if(cmd == NULL) {
fprintf(stderr,"Allocating distribute command string memory failed!\n");
return 1;
}
sprintf(cmd,"chirp_distribute -a hostname -X %s %s %s",hn,remote_dir,node_list);
debug(D_CHIRP,"%s\n",cmd);
system(cmd);
sprintf(cmd,"chirp_distribute -a hostname -X %s %s %s",hn,remote_dir,hn);
debug(D_CHIRP,"%s\n",cmd);
system(cmd);
free(cmd);
cmd=NULL;
}
// next, delete matrix if desired.
if(rm_mat) {
fprintf(stderr,"Remove Matrix State Mode\n");
chirp_matrix_delete( mat_host, mat_path, time(0)+600 );
}
// next, delete local if desired.
if(rm_local) {
fprintf(stderr,"Remove Local State Mode\n");
cmd = (char *) malloc(strlen("rm -rf ")+1+strlen(local_dir)+1);
if(cmd == NULL) {
fprintf(stderr,"Allocating distribute command string memory failed!\n");
return 1;
}
sprintf(cmd,"rm -rf %s",local_dir);
system(cmd);
free(cmd);
cmd=NULL;
}
return 0;
}
struct chirp_matrix *chirp_matrix_create(const char *host, const char *path, int width, int height, int element_size, int nhosts, time_t stoptime)
{
char host_file[CHIRP_LINE_MAX];
int result;
unsigned int i;
char **hosts;
int nfiles = nhosts;
while(1) {
INT64_T n_row_per_file = height / nfiles;
if(height % nfiles)
n_row_per_file++;
INT64_T file_size = n_row_per_file * width * element_size;
if(file_size > GIGABYTE) {
nfiles *= 2;
continue;
} else {
break;
}
}
char line[CHIRP_LINE_MAX * nfiles];
FILE *file = NULL;
if(getenv("CHIRP_HOSTS")) {
sprintf(host_file, "%s", getenv("CHIRP_HOSTS"));
file = fopen(host_file, "r");
}
if(!file) {
if(getenv("HOME")) {
sprintf(host_file, "%s/.chirp/hosts", getenv("HOME"));
file = fopen(host_file, "r");
}
}
if(!file) {
sprintf(host_file, "./chirp_hosts");
file = fopen(host_file, "r");
if(!file) {
file = fopen(host_file, "w");
char hostname[CHIRP_LINE_MAX];
gethostname(hostname, CHIRP_LINE_MAX - 1); // get hostname, this may not have domain name, though!
fprintf(file, "%s\n", hostname);
fclose(file);
file = fopen(host_file, "r");
}
}
if(!file) {
debug(D_NOTICE | D_CHIRP, "matrix: could not open host list in %s: %s\n", host_file, strerror(errno));
errno = EINVAL;
return 0;
}
hosts = malloc(sizeof(*hosts) * nhosts);
for(i = 0; (int) i < nhosts; i++) {
if(!fgets(line, sizeof(line), file)) {
rewind(file);
fgets(line, sizeof(line), file);
}
hosts[i] = strdup(line);
int len = strlen(hosts[i]);
hosts[i][len - 1] = '\0';
}
fclose(file);
sprintf(line, "%d\n%d\n%d\n%d\n%d\n", width, height, element_size, nhosts, nfiles);
char datapath1[CHIRP_LINE_MAX];
char datapath2[CHIRP_LINE_MAX];
char datapath3[CHIRP_LINE_MAX];
char username[USERNAME_MAX];
char cookie[16];
username_get(username);
string_cookie(cookie, sizeof(cookie));
sprintf(datapath1, "/%s", username);
sprintf(datapath2, "/%s/matrixdata", username);
sprintf(datapath3, "/%s/matrixdata/%s", username, cookie);
for(i = 0; (int) i < nfiles; i++) {
const char *datahost = hosts[i % nhosts];
result = chirp_reli_mkdir(datahost, datapath1, 0700, stoptime);
result = chirp_reli_mkdir(datahost, datapath2, 0700, stoptime);
result = chirp_reli_mkdir(datahost, datapath3, 0700, stoptime);
sprintf(&line[strlen(line)], "%s %s/data.%d\n", datahost, datapath3, i);
}
for(i = 0; (int) i < nhosts; i++) {
free(hosts[i]);
}
free(hosts);
char metapath[CHIRP_LINE_MAX];
strcpy(metapath, path);
result = chirp_reli_putfile_buffer(host, path, line, 0700, strlen(line), stoptime);
if(result < 0) {
for(i = 1; i < strlen(metapath); i++)
if(metapath[i] == '/') {
metapath[i] = '\0';
result = chirp_reli_mkdir(host, metapath, 0700, stoptime);
if(result < 0 && errno != EEXIST) {
debug(D_CHIRP, "matrix: could not build directory /chirp/%s/%s to create metadata file: %s\n", host, metapath, strerror(errno));
return 0;
}
metapath[i] = '/';
}
result = chirp_reli_putfile_buffer(host, path, line, 0700, strlen(line), stoptime);
if(result < 0) {
debug(D_CHIRP, "matrix: could not create metadata file /chirp/%s/%s: %s\n", host, path, strerror(errno));
return 0;
}
}
debug(D_CHIRP, "matrix: created matrix %s/%s -- now opening\n", host, path);
return chirp_matrix_open(host, path, stoptime);
}
