// write output to .clu file - with 1 added to cluster numbers, and empties removed.
void SaveOutput(Array<int> &OutputClass) {
int p, c;
char fname[STRLEN];
FILE *fp;
int MaxClass = 0;
Array<int> NotEmpty(MaxPossibleClusters);
Array<int> NewLabel(MaxPossibleClusters);
// find non-empty clusters
for(c=0;c<MaxPossibleClusters;c++) NewLabel[c] = NotEmpty[c] = 0;
for(p=0; p<OutputClass.size(); p++) NotEmpty[OutputClass[p]] = 1;
// make new cluster labels so we don't have empty ones
NewLabel[0] = 1;
MaxClass = 1;
for(c=1;c<MaxPossibleClusters;c++) {
if (NotEmpty[c]) {
MaxClass++;
NewLabel[c] = MaxClass;
}
}
// print file
sprintf(fname, "%s.clu.%d", FileBase, ElecNo);
fp = fopen_safe(fname, "w");
fprintf(fp, "%d\n", MaxClass);
for (p=0; p<OutputClass.size(); p++) fprintf(fp, "%d\n", NewLabel[OutputClass[p]]);
fclose(fp);
}
int main(void)
{
/* Setup */
identifier = (int *)calloc_safe(1, sizeof(int));
*identifier = PARENT_IDENTIFIER;
process_number = -1;
/* Open file for reading */
FILE *input_file = fopen_safe("input.txt", "r");
/* Create necessary pipes */
create_pipes();
/* Fork children processes */
pid_t main_id = getpid(); // main_id contains parent process ID
create_mappers(main_id);
if (getpid() == main_id) // more efficient to include this statement
create_reducers(main_id);
/* Close proper pipe ends */
close_pipe_ends();
/* Parse the input file. Processes handle properly. */
parse_file(input_file);
// free heap ptrs
return 0;
}
void open_in_file(FILE ** in, char *infile_name)
{
*in = NULL;
if (infile_name == NULL)
*in = stdin;
else
*in = fopen_safe(infile_name, "rb");
}
void open_out_file(FILE ** out, char *outfile_name)
{
*out = NULL;
if (global_options.use_stdout)
*out = stdout;
else if (outfile_name != NULL)
*out = fopen_safe(outfile_name, "wb");
else if (!isatty(fileno(stdout)) || global_options.force)
*out = stdout;
else {
log_print(WARN, "igzip: No output location. Use -c to output to terminal\n");
exit(FILE_OPEN_ERROR);
}
}
void profileWrite (GC_state s, GC_profileData p, const char *fileName) {
FILE *f;
const char* kind;
if (DEBUG_PROFILE)
fprintf (stderr, "profileWrite("FMTPTR",%s)\n", (uintptr_t)p, fileName);
f = fopen_safe (fileName, "wb");
writeString (f, "MLton prof\n");
switch (s->profiling.kind) {
case PROFILE_ALLOC:
kind = "alloc\n";
break;
case PROFILE_COUNT:
kind = "count\n";
break;
case PROFILE_NONE:
die ("impossible PROFILE_NONE");
// break;
case PROFILE_TIME_FIELD:
kind = "time\n";
break;
case PROFILE_TIME_LABEL:
kind = "time\n";
break;
default:
kind = "";
assert (FALSE);
}
writeString (f, kind);
writeString (f, s->profiling.stack ? "stack\n" : "current\n");
writeUint32X (f, s->magic);
writeNewline (f);
writeUintmaxU (f, p->total);
writeString (f, " ");
writeUintmaxU (f, p->totalGC);
writeNewline (f);
writeUint32U (f, s->sourceMaps.sourcesLength);
writeNewline (f);
for (GC_sourceIndex i = 0; i < s->sourceMaps.sourcesLength; i++)
writeProfileCount (s, f, p,
(GC_profileMasterIndex)i);
writeUint32U (f, s->sourceMaps.sourceNamesLength);
writeNewline (f);
for (GC_sourceNameIndex i = 0; i < s->sourceMaps.sourceNamesLength; i++)
writeProfileCount (s, f, p,
(GC_profileMasterIndex)(i + s->sourceMaps.sourcesLength));
fclose_safe (f);
}
void SetupParams(int argc, char **argv) {
char fname[STRLEN];
init_params(argc, argv);
// PARAMETER DEFINITIONS GO HERE
STRING_PARAM(FileBase);
INT_PARAM(ElecNo);
INT_PARAM(MinClusters);
INT_PARAM(MaxClusters);
INT_PARAM(MaxPossibleClusters);
INT_PARAM(nStarts);
INT_PARAM(RandomSeed);
BOOLEAN_PARAM(Debug);
INT_PARAM(Verbose);
STRING_PARAM(UseFeatures);
INT_PARAM(DistDump);
FLOAT_PARAM(DistThresh);
INT_PARAM(FullStepEvery);
FLOAT_PARAM(ChangedThresh);
BOOLEAN_PARAM(Log);
BOOLEAN_PARAM(Screen);
INT_PARAM(MaxIter);
STRING_PARAM(StartCluFile);
INT_PARAM(SplitEvery);
FLOAT_PARAM(PenaltyMix);
INT_PARAM(Subset);
if (argc<3) {
fprintf(stderr, "Usage: KlustaKwik FileBase ElecNo [Arguments]\n\n");
fprintf(stderr, "Default Parameters: \n");
print_params(stderr);
exit(1);
}
strcpy(FileBase, argv[1]);
ElecNo = atoi(argv[2]);
if (Screen) print_params(stdout);
// open log file, if required
if (Log) {
sprintf(fname, "%s.klg.%d", FileBase, ElecNo);
logfp = fopen_safe(fname, "w");
print_params(logfp);
}
}
// LoadClu(CluFile)
void KK::LoadClu(char *CluFile) {
FILE *fp;
int p, c, val;
int status;
fp = fopen_safe(CluFile, "r");
status = fscanf(fp, "%d", &nStartingClusters);
nClustersAlive = nStartingClusters;// -1;
for(c=0; c<MaxPossibleClusters; c++) ClassAlive[c]=(c<nStartingClusters);
for(p=0; p<nPoints; p++) {
status = fscanf(fp, "%d", &val);
if (status==EOF) Error("Error reading cluster file");
Class[p] = val-1;
}
}
// Loads in Fet file. Also allocates storage for other arrays
void KK::LoadData(char *FileBase, int ElecNo, char *UseFeatures)
{
char fname[STRLEN];
//char fnamemask[STRLEN];
char fnamefmask[STRLEN];
char line[STRLEN];
int p, i, j;
int nFeatures, nmaskFeatures; // not the same as nDims! we don't use all features.
FILE *fp;
//FILE *fpmask;
FILE *fpfmask;
int status;
//int maskstatus;
scalar val;
//int maskval;
int UseLen;
scalar max, min;
//bool usemasks = (UseDistributional && !UseFloatMasks);
// open file
sprintf(fname, "%s.fet.%d", FileBase, ElecNo);
fp = fopen_safe(fname, "r");
//if(usemasks)
//{
// sprintf(fnamemask,"%s.mask.%d", FileBase, ElecNo);
// fpmask = fopen_safe(fnamemask, "r");
//} else
//{
// fpmask = NULL;
//}
if((MaskStarts > 0)&& UseDistributional)
{
Output("-------------------------------------------------------------------------");
Output("\nUsing Distributional EM with Maskstarts\n");
MinClusters = MaskStarts;
MaxClusters = MaskStarts;
Output("NOTE: Maskstarts overides above values of MinClusters and MaxClusters \
\nMinClusters = %d \nMaxClusters = %d \n ", MinClusters,MaxClusters);
}
int main_app(int argc, char* argv[])
{
char name[80] = {"../../work/data/paris"};
int l1 = 352;
int l2 = 288;
int k=1;
if (k < argc) sprintf(name, "%s", argv[k++]);
if (k < argc) l1 = atoi(argv[k++]);
if (k < argc) l2 = atoi(argv[k++]);
int32_t qp = 16 + 6*2;
int32_t keyint = 50;
int32_t bframe = 0;
int l = l1*l2;
int m = l>>2;
int lmm = l + m + m;
char filename[80];
char filename_f_yuv[80];
//char filename_h_yuv[80];
char filename_g_264[80];
char filename_g_yuv[80];
FILE* file_f_yuv;
//FILE* file_h_yuv;
FILE* file_g_264;
//FILE* file_g_yuv;
int g_264_size = 0;
void* mcdull = mcdull_new();
int ret = 0;
sprintf(filename, "%s_%dx%d", name, l1, l2);
sprintf(filename_f_yuv, "%s.yuv", filename);
//sprintf(filename_h_yuv, "%s_h.yuv", filename);
#if mcdull_SWATAW
sprintf(filename_g_264, "%s_mcdull.264", filename);
sprintf(filename_g_yuv, "%s_mcdull.yuv", filename);
//sprintf(filename_g_mp4, "%s_mcdull.mp4", filename);
#else
sprintf(filename_g_264, "%s_x264.264", filename);
sprintf(filename_g_yuv, "%s_x264.yuv", filename);
//sprintf(filename_g_mp4, "%s_x264.mp4", filename);
#endif
mcdull_set_qp(mcdull, qp);
mcdull_set_keyint(mcdull, keyint);
mcdull_set_bframe(mcdull, bframe);
ret = mcdull_open(mcdull, l1, l2); if (ret < 0) return -1;
ret = fopen_safe(&file_f_yuv, filename_f_yuv, "rb"); if (ret != 0) return -1;
//ret = fopen_safe(&file_h_yuv, filename_h_yuv, "wb"); if (ret != 0) return -1;
ret = fopen_safe(&file_g_264, filename_g_264, "wb"); if (ret != 0) return -1;
//ret = fopen_safe(&file_g_yuv, filename_g_yuv, "wb"); if (ret != 0) return -1;
int i_frame_total = FRAME_TOTAL;
int i_frame, i_frame_output;
for( i_frame = 0, i_frame_output = 0; i_frame < i_frame_total; i_frame++ )
{
ret = fread (f_yuv, 1, lmm, file_f_yuv); if (ret < 0) goto done;
//ret = fwrite(f_yuv, 1, lmm, file_h_yuv); if (ret < 0) goto done;
g_264_size = mcdull_encode(mcdull, f_yuv, i_frame, &g_264[1], &g_264[0], g_yuv);
{ int i; for (i=0; i<80; i++) printf("\b"); }
printf("%4d: size = %6d", i_frame, g_264_size);
if (g_264_size < 0)
return -1;
if (g_264_size > 0)
{
ret = fwrite(&g_264[1], 1, g_264_size, file_g_264); if (ret < 0) goto done;
//ret = fwrite(&g_yuv[0], 1, lmm, file_g_yuv); if (ret < 0) goto done;
i_frame_output++;
}
}
do
{
g_264_size = mcdull_encode_flush(mcdull, &g_264[1], &g_264[0], g_yuv);
{ int i; for (i=0; i<80; i++) printf("\b"); }
printf("%4d: size = %6d", i_frame, g_264_size);
if (g_264_size > 0)
{
ret = fwrite(&g_264[1], 1, g_264_size, file_g_264); if (ret < 0) goto done;
//ret = fwrite(&g_yuv[0], 1, lmm, file_g_yuv); if (ret < 0) goto done;
i_frame_output++;
}
i_frame++;
} while( g_264_size > 0 );
done:
printf("\n");
mcdull_close(mcdull);
mcdull_delete(mcdull);
fclose(file_f_yuv);
//fclose(file_h_yuv);
fclose(file_g_264);
//fclose(file_g_yuv);
return 0;
}
// Loads in Fet file. Also allocates storage for other arrays
void KK::LoadData() {
char fname[STRLEN];
char line[STRLEN];
int p, i, j;
int nFeatures; // not the same as nDims! we don't use all features.
FILE *fp;
int status;
float val;
int UseLen;
float max, min;
// open file
sprintf(fname, "%s.fet.%d", FileBase, ElecNo);
fp = fopen_safe(fname, "r");
// count lines;
nPoints=-1; // subtract 1 because first line is number of features
while(fgets(line, STRLEN, fp)) {
nPoints++;
}
// rewind file
fseek(fp, 0, SEEK_SET);
// read in number of features
fscanf(fp, "%d", &nFeatures);
// calculate number of dimensions
UseLen = strlen(UseFeatures);
nDims=0;
for(i=0; i<nFeatures; i++) {
nDims += (i<UseLen && UseFeatures[i]=='1');
}
AllocateArrays();
// load data
for (p=0; p<nPoints; p++) {
j=0;
for(i=0; i<nFeatures; i++) {
status = fscanf(fp, "%f", &val);
if (status==EOF) Error("Error reading feature file");
if (i<UseLen && UseFeatures[i]=='1') {
Data[p*nDims + j] = val;
j++;
}
}
}
fclose(fp);
// normalize data so that range is 0 to 1: This is useful in case of v. large inputs
for(i=0; i<nDims; i++) {
//calculate min and max
min = HugeScore; max=-HugeScore;
for(p=0; p<nPoints; p++) {
val = Data[p*nDims + i];
if (val > max) max = val;
if (val < min) min = val;
}
// now normalize
for(p=0; p<nPoints; p++) Data[p*nDims+i] = (Data[p*nDims+i] - min) / (max-min);
}
Output("Loaded %d data points of dimension %d.\n", nPoints, nDims);
}