// Store data chunk generated by memory hooks
void DataChunStorage::storeDataChunk(void *dataChunk)
{
while(true) // infinite loop until passed chunk is stored in chache
{
int currentCache = cacheIndex.load();
int indexInsideCache = indexStoreing[currentCache].fetch_add(1);
if(indexInsideCache < CACHE_SIZE)
{
dataCache[currentCache][indexInsideCache] = *(DataChunkBase*)dataChunk;
if(indexStored[currentCache].fetch_add(1) == CACHE_SIZE-1)
{
writeCache(currentCache, CACHE_SIZE);
indexStored[currentCache] = 0;
indexStoreing[currentCache] = 0;
}
return;
}
else if(indexInsideCache == CACHE_SIZE) // Index is now pointing out of the array
{
int nextCacheIndex = currentCache+1;
if(nextCacheIndex >= NUMBER_OF_CACHES)
nextCacheIndex++;
while(indexStoreing[nextCacheIndex].load() != 0)
; // Wait until data from next cache are written
// Increment cache number
cacheIndex.store(nextCacheIndex);
}
}
}
/*
** Sync the file. First flush the write-cache to disk, then call the
** real sync() function.
*/
int sqlite3OsSync(OsFile *id){
int rc;
/* printf("SYNC %s (%d blocks)\n", (*id)->zName, (*id)->nBlk); */
rc = writeCache(*id);
if( rc!=SQLITE_OK ) return rc;
rc = sqlite3RealSync(&(*id)->fd);
return rc;
}
/**
* Read in a character to be read by the command handler
* A terminator character prompts the buffer to be processed
*/
void CommandHandler::readIn(char inChar) {
if (inChar == _terminator){ // Check for the terminator (default '\r') meaning end of command
processCommands();
}
else{
writeCache(inChar);
}
}
/*
** Close the file.
*/
int sqlite3OsClose(OsFile *id){
if( !(*id) ) return SQLITE_OK;
if( (*id)->fd.isOpen ){
/* printf("CLOSE %s (%d blocks)\n", (*id)->zName, (*id)->nBlk); */
writeCache(*id);
sqlite3RealClose(&(*id)->fd);
}
closeFile(id);
return SQLITE_OK;
}
void DataChunStorage::deinitDataChunkStorage()
{
// Write rest of data in to file
for(int i = 0; i < NUMBER_OF_CACHES; i++)
{
if(indexStored[i].load() > 0)
{
writeCache(i,indexStored[i].load()-1);
}
}
close(logFileFd);
}
int
main (int argc, char** argv)
{
// const char trace = 0;
// const unsigned debug = 0;
const char *me = "main";
// parse and validate the command line options
struct paramsStruct params;
parseCommandLine(argc, argv, ¶ms);
// create results directory in ANALYSIS directory
// permissions are read, write for the owner
char resultsDirectoryName[256] = "";
makeResultsDirectory(resultsDirectoryName,
sizeof(resultsDirectoryName),
¶ms);
// start logging
char logFilePath[256] = "";
{
const int outputLength = snprintf(logFilePath,
sizeof(logFilePath),
"%s/run.log",
resultsDirectoryName);
if (outputLength > (int) sizeof(logFilePath)) {
fprintf(stderr, "%s: logFilePath too small", me);
exit(1);
}
}
Log_T log = Log_new(logFilePath, stderr);
// log the command line parameters
LOG(log,"started log file %s\n", logFilePath);
LOG(log,"params: algo=%s\n", params.algo);
LOG(log," : obs=%s\n", params.obs);
LOG(log," : radius=%d\n", params.radius);
LOG(log," : which=%s\n", params.which);
// check the command line parameters
assert(strcmp(params.algo, "knn") == 0);
assert(strcmp(params.obs, "1A") == 0);
// read the input files
const unsigned nObservations = 217376; // adjust of OBS != 1A
const unsigned nFeatures = 55;
double *apns = readCsvNoHeader(nObservations, "aps.csv");
double *dates = readCsvNoHeader(nObservations, "date.csv");
char *featuresHeaderP;
double *features = readFeatures(nObservations,
nFeatures,
&featuresHeaderP);
double *prices = readCsvNoHeader(nObservations, "SALE-AMOUNT-log.csv");
// convert dates to days past the epoch
unsigned dayStdColumn = 5; // the 6th column contains the standardized day value
assert(columnHeaderEqual(featuresHeaderP, dayStdColumn, "day-std"));
double *days = convertDatesToDays(nObservations, dates);
free(dates);
double mean;
double stdv;
determineMeanStdv(nObservations, days, &mean, &stdv);
double *daysStd = standardize(nObservations, days, mean, stdv);
replaceDay(nObservations, nFeatures, features, daysStd, dayStdColumn);
free(days);
free(daysStd);
// generate one set of estimates
FILE *resultFile;
{
char resultFilePath[256];
const int outputLength =
snprintf(resultFilePath,
sizeof(resultFilePath),
"%s/estimates-laufer.csv",
resultsDirectoryName);
if (outputLength > (int) sizeof(resultFilePath)) {
fprintf(stderr, "%s: resultFilePath too small", me);
exit(1);
}
LOG(log, " result file path: %s\n", resultFilePath);
resultFile = fopen(resultFilePath, "w");
}
assert(resultFile);
if (strcmp(params.which, "laufer"))
createLaufer(nObservations, nFeatures,
apns, dates, features, prices,
log,
resultFile);
else
assert(NULL != "logic error");
// OLD CODE BELOW THIS LINE
#if 0
double **pricesHatP = NULL;
if (params.useCache)
pricesHatP = readCache(nObservations, params.obs, log, kMax);
// determine estimated prices for any missing entries in the cache
// this operation could be fast or very slow
// MAYBE: write out cache periodically
const unsigned cacheMutated = completeCache(nObservations,
pricesHatP,
params.obs,
log,
kMax,
pricesP,
debug);
if (params.useCache && cacheMutated)
writeCache(nObservations, pricesHatP, params.obs, log, kMax);
// select which set of estimates to create
if (paramsP->whichIsLaufer)
createEstimatesLaufer(nObservations, nFeatures,
features, dates, prices);
else
assert(false); // should never get here
// pricesHatP[i][k] is
// the estimate priced of transaction indexed i for k nearest neighbors
// for each value of k, determine RMSE overall all the test transactions
// determine kArgMin, the k providing the lowest RMSE
// write CSV containing <k, rmse> values
char resultFilePath[256];
{
const int outputLength =
snprintf(resultFilePath,
sizeof(resultFilePath),
"%s/k-rmse.csv",
directoryName);
if (outputLength > (int) sizeof(resultFilePath)) {
fprintf(stderr, "%s: resultFilePath too small", me);
exit(1);
}
LOG(log, " result file path: %s\n", resultFilePath);
}
FILE *resultFile = fopen(resultFilePath, "w");
assert(resultFile);
// log best k for random sample of test observations
bestK(0.01, nObservations, pricesHatP, pricesP, log, kMax);
// write CSV header
fprintf(resultFile, "k,rmse\n");
unsigned kArgMin = 0;
double lowestRMSE = DBL_MAX;
for (unsigned hpK = 0; hpK < kMax; hpK++) {
// determine rmse for this k
const double rmse = determineRmse(nObservations, pricesHatP, pricesP, hpK);
// check if we have a new best k
LOG(log, "hpK %u rmse %f\n", hpK + 1, rmse);
fprintf(resultFile, "%u,%f\n", hpK + 1, rmse);
if (rmse < lowestRMSE) {
lowestRMSE = rmse;
kArgMin = hpK;
}
}
#endif
//
LOG(log, "%s\n", "finished");
exit(0);
}
