unsigned int HSPParseFASTAToPacked(const char* FASTAFileName, const char* annotationFileName, const char* packedDNAFileName, const char* ambiguityFileName, const unsigned int FASTARandomSeed, const int maskLowerCase) {
FILE *FASTAFile, *annotationFile, *packedDNAFile, *ambiguityFile;
NewAnnotation *chrAnnotation;
int chrAnnAllocated = 256;
int blockAllocated = 256;
char c;
int chrNum, blockNum;
unsigned int i, l;
int nCount;
unsigned int chrLen, usefulCharNum, numCharInBuffer, totalNumChar;
unsigned char charMap[255];
char *chrSeq, *p;
unsigned int chrAllocated = 65536;
unsigned char buffer[PACKED_BUFFER_SIZE];
unsigned char packedBuffer[PACKED_BUFFER_SIZE / 4];
chrLen = usefulCharNum = numCharInBuffer = totalNumChar = chrNum = blockNum = i = l = nCount = 0;
FASTAFile = (FILE*)fopen64(FASTAFileName, "r");
if (FASTAFile == NULL) {
fprintf(stderr, "ParseFASTToPacked() : Cannot open FASTAFileName!\n");
exit(1);
}
annotationFile = (FILE*)fopen64(annotationFileName, "w");
if (annotationFile == NULL) {
fprintf(stderr, "ParseFASTToPacked() : Cannot open annotationFileName!\n");
exit(1);
}
packedDNAFile = (FILE*)fopen64(packedDNAFileName, "wb");
if (packedDNAFile == NULL) {
fprintf(stderr, "ParseFASTToPacked() : Cannot open packedDNAFileName!\n");
exit(1);
}
ambiguityFile = (FILE*)fopen64(ambiguityFileName, "w");
if (ambiguityFile == NULL) {
fprintf(stderr, "ParseFASTToPacked() : Cannot open ambiguityFileName!\n");
exit(1);
}
HSPFillCharMap(charMap);
c = (char)getc(FASTAFile);
if (c != '>') {
fprintf(stderr, "ParseFASTToPacked() : FASTA file does not begin with '>'!\n");
exit(1);
}
chrAnnotation = (NewAnnotation *)malloc(sizeof(NewAnnotation)*chrAnnAllocated);
chrSeq = (char*)malloc(sizeof(char)*chrAllocated);
chrNum = blockNum = usefulCharNum = numCharInBuffer = 0;
while(!feof(FASTAFile)){
if (feof(FASTAFile)) break;
if (chrNum == chrAnnAllocated){
chrAnnAllocated <<= 1;
chrAnnotation = (NewAnnotation *)realloc(chrAnnotation, sizeof(NewAnnotation)*chrAnnAllocated);
// printf("%d\n", chrNum);
}
l=0;
c = (char)getc(FASTAFile);
while(!feof(FASTAFile) && c!='\t' && c!=' ' && c!='\n' && l<MAX_SEQ_NAME_LENGTH){
chrAnnotation[chrNum].chrName[l]=c;
l++;
c=(char)getc(FASTAFile);
}
chrAnnotation[chrNum].chrName[l]='\0';
while(c!='\n'){
c=(char)getc(FASTAFile);
}
chrLen = 0;
while(c!='>' && !feof(FASTAFile)){
if (c!='\n'){
if (c>='a' && c<='z'){
c+='A'-'a';
}
if (chrLen >= chrAllocated){
chrAllocated <<= 1;
chrSeq = (char*)realloc(chrSeq, sizeof(char)*chrAllocated);
}
*(chrSeq+chrLen) = c;
chrLen += 1;
}
c=(char)getc(FASTAFile);
}
if (chrLen < 75) continue;
//*
i=0;
p=chrSeq;
while (ambiguityCount[charMap[(int)*p]] == 1 && i++ != chrLen) p++;
if (i == chrLen) {
blockNum = 1;
chrAnnotation[chrNum].blockInChr = (ChrBlock *)malloc(sizeof(ChrBlock)*blockNum);
chrAnnotation[chrNum].chrStart = usefulCharNum;
chrAnnotation[chrNum].blockNum = blockNum;
chrAnnotation[chrNum].blockInChr[0].blockStart = usefulCharNum;
chrAnnotation[chrNum].blockInChr[0].ori = 0;
usefulCharNum += chrLen;
chrAnnotation[chrNum].chrEnd = usefulCharNum-1;
chrAnnotation[chrNum].blockInChr[0].blockEnd = usefulCharNum-1;
i=0;
while(i<chrLen){
if (numCharInBuffer >= PACKED_BUFFER_SIZE) {
ConvertTextToBytePacked(buffer, packedBuffer, charMap, 4, PACKED_BUFFER_SIZE);
fwrite(packedBuffer, 1, PACKED_BUFFER_SIZE / 4, packedDNAFile);
numCharInBuffer = 0;
}
buffer[numCharInBuffer++] = chrSeq[i++];
}
} else {
i=0;
p = chrSeq;
while (ambiguityCount[charMap[(int)*p]]!=1 && ++i!=chrLen) p++;
if (i<10) { i = 0; p = chrSeq;}
blockNum = 1;
chrAnnotation[chrNum].blockInChr = (ChrBlock *)malloc(sizeof(ChrBlock)*blockAllocated);
chrAnnotation[chrNum].chrStart = usefulCharNum;
chrAnnotation[chrNum].blockInChr[blockNum-1].ori = i;
chrAnnotation[chrNum].blockInChr[blockNum-1].blockStart = usefulCharNum;
int len=0;
while (i<chrLen) {
if(ambiguityCount[charMap[(int)*p]] == 1){
if (numCharInBuffer >= PACKED_BUFFER_SIZE) {
ConvertTextToBytePacked(buffer, packedBuffer, charMap, 4, PACKED_BUFFER_SIZE);
fwrite(packedBuffer, 1, PACKED_BUFFER_SIZE / 4, packedDNAFile);
numCharInBuffer = 0;
}
buffer[numCharInBuffer++] = *p++;
i++;
usefulCharNum++;
len++;
}else{
nCount = 0;
while((ambiguityCount[charMap[(int)*p]]!=1) && i<chrLen){
nCount++;
i++;
p++;
}
if (nCount<10) {
do {
if (numCharInBuffer >= PACKED_BUFFER_SIZE) {
ConvertTextToBytePacked(buffer, packedBuffer, charMap, 4, PACKED_BUFFER_SIZE);
fwrite(packedBuffer, 1, PACKED_BUFFER_SIZE / 4, packedDNAFile);
numCharInBuffer = 0;
}
buffer[numCharInBuffer++] = 'G';
usefulCharNum++;
len++;
} while(--nCount>0);
} else {
if (i<chrLen) {
chrAnnotation[chrNum].blockInChr[blockNum-1].blockEnd = usefulCharNum -1;
chrAnnotation[chrNum].blockInChr[blockNum-1].ori = i-nCount-len;
if (blockNum == blockAllocated){
blockAllocated <<= 1;
chrAnnotation[chrNum].blockInChr = (ChrBlock *)realloc(chrAnnotation[chrNum].blockInChr, sizeof(ChrBlock)*blockAllocated);
}
blockNum++;
len=0;
chrAnnotation[chrNum].blockInChr[blockNum-1].blockStart = usefulCharNum;
} else {
i-=nCount;
break;
}
}
}
}
chrAnnotation[chrNum].blockInChr[blockNum-1].blockEnd = usefulCharNum-1;
chrAnnotation[chrNum].blockInChr[blockNum-1].ori = i-len;
chrAnnotation[chrNum].blockNum = blockNum;
chrAnnotation[chrNum].chrEnd = usefulCharNum-1;
}
//*/
chrNum++;
totalNumChar += chrLen;
}
if (numCharInBuffer > 0) {
ConvertTextToBytePacked(buffer, packedBuffer, charMap, 4, numCharInBuffer);
fwrite(packedBuffer, 1, (numCharInBuffer + 3) / 4, packedDNAFile);
numCharInBuffer = 0;
}
if (totalNumChar % 4 == 0) {
c = 0;
fwrite(&c, 1, 1, packedDNAFile);
}
c = (char)(totalNumChar % 4);
fwrite(&c, 1, 1, packedDNAFile);
fclose(packedDNAFile);
fprintf(annotationFile, "%u\t%d\t%d\n", totalNumChar, chrNum, FASTARandomSeed);
int j=0;
int total = 0;
for (i=0;i<chrNum;i++) {
fprintf(annotationFile, "%d\t%s\n", (int)strlen(chrAnnotation[i].chrName), chrAnnotation[i].chrName);
total += chrAnnotation[i].blockNum;
}
fprintf(annotationFile, "%d\n", total);
// fprintf(stderr, "total block, %d, %d, %d\n",i,chrNum, total);
for(i=0;i<chrNum;i++){
for(j=0;j<chrAnnotation[i].blockNum;j++){
fprintf(annotationFile,"%d\t%u\t%u\t%u\n",i, chrAnnotation[i].blockInChr[j].blockStart, chrAnnotation[i].blockInChr[j].blockEnd, chrAnnotation[i].blockInChr[j].ori);
}
free(chrAnnotation[i].blockInChr);
}
free(chrAnnotation);
fclose(annotationFile);
return chrNum;
}
int DownloadPartition (libusb_device_handle *handle, char *fid,
uint32_t sector, uint32_t sectors = 0)
{
char *buf = (char*) malloc (65536);
FILE *fin;
off64_t file_size;
uint file_sectors;
uint nand_sec_size = 512;
uint usb_rec_size, usb_rec_secs;
uint read_bytes, read_secs, bytes_read;
uint sector_key, sector_limit;
uint usb_flags;
usb_rec_size = 65536;
usb_rec_secs = usb_rec_size / nand_sec_size;
usb_rec_size = usb_rec_secs * nand_sec_size;
if (NULL == (fin = fopen64 (fid, "rb"))) {
perror("Failed to open file to send: ");
exit(1);
}
// aw_fes_send_file (handle, MagicAddr, AW_FEL2_DRAM, FN->magic_cr_start); // not when A10
printf ("Sending %s...", fid);
fflush (stdout);
fseeko64 (fin, 0, SEEK_END);
file_size = ftello64 (fin);
file_sectors = (file_size + nand_sec_size - 1) / nand_sec_size;
fseeko64 (fin, 0, SEEK_SET);
sector_key = sector;
if (sectors < file_sectors)
sectors = file_sectors;
sector_limit = sector_key + sectors;
while (sector_key < sector_limit)
{
read_secs = sector_limit - sector_key;
if (read_secs > usb_rec_secs)
read_secs = usb_rec_secs;
read_bytes = read_secs * nand_sec_size;
bytes_read = fread (buf, 1, read_bytes, fin);
if (bytes_read < read_bytes)
PutNulls (buf + bytes_read, read_bytes - bytes_read);
usb_flags = AW_FEL2_NAND | AW_FEL2_WR;
if (sector_key == sector)
usb_flags |= AW_FEL2_FIRST;
if (sector_key + read_secs == sector_limit)
usb_flags |= AW_FEL2_LAST;
aw_fes_write (handle, sector_key, buf, read_bytes, usb_flags);
//printf ("sector = %d, sector_key = %d, read_secs = %d, sector_limit = %d, flags = %x\n", sector, sector_key, read_secs, sector_limit, usb_flags);
sector_key += read_secs;
}
printf ("done\n");
fclose (fin);
// aw_fes_send_file (handle, MagicAddr, AW_FEL2_DRAM, FN->magic_cr_end); // not when A10
free (buf);
return 0;
}
int http_put (const char * file_path, const char * url, const char * login, const char * password)
{
int code = ERROR;
if (curl_initialized!=1) {
curl_global_init(CURL_GLOBAL_SSL);
curl_initialized = 1;
}
struct stat64 mystat;
if (stat64 (file_path, &mystat)) {
logprintfl (EUCAERROR, "http_put(): failed to stat %s\n", file_path);
return code;
}
if (!S_ISREG(mystat.st_mode)) {
logprintfl (EUCAERROR, "http_put(): %s is not a regular file\n", file_path);
return code;
}
FILE * fp = fopen64 (file_path, "r");
if (fp==NULL) {
logprintfl (EUCAERROR, "http_put(): failed to open %s for reading\n", file_path);
return code;
}
CURL * curl;
CURLcode result;
curl = curl_easy_init ();
if (curl==NULL) {
logprintfl (EUCAERROR, "http_put(): could not initialize libcurl\n");
fclose (fp);
return code;
}
logprintfl (EUCAINFO, "http_put(): uploading %s\n", file_path);
logprintfl (EUCAINFO, " to %s\n", url);
char error_msg [CURL_ERROR_SIZE];
curl_easy_setopt (curl, CURLOPT_ERRORBUFFER, error_msg);
curl_easy_setopt (curl, CURLOPT_URL, url);
curl_easy_setopt (curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt (curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)mystat.st_size);
curl_easy_setopt (curl, CURLOPT_SSL_VERIFYPEER, 0L); // TODO: make this optional?
curl_easy_setopt (curl, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt (curl, CURLOPT_LOW_SPEED_LIMIT, 360L); // must have at least a 360 baud modem
curl_easy_setopt (curl, CURLOPT_LOW_SPEED_TIME, 10L); // abort if below speed limit for this many seconds
if (login!=NULL && password!=NULL) {
char userpwd [STRSIZE];
snprintf (userpwd, STRSIZE, "%s:%s", login, password);
curl_easy_setopt (curl, CURLOPT_USERPWD, userpwd);
}
struct read_request params;
params.fp = fp;
params.timestamp = time(NULL);
params.file_size = (long long)mystat.st_size;
curl_easy_setopt (curl, CURLOPT_READDATA, ¶ms);
curl_easy_setopt (curl, CURLOPT_READFUNCTION, read_data);
int retries = TOTAL_RETRIES;
int timeout = FIRST_TIMEOUT;
do {
params.total_read = 0L;
params.total_calls = 0L;
result = curl_easy_perform (curl); /* do it */
logprintfl (EUCADEBUG, "http_put(): uploaded %ld bytes in %ld sends\n", params.total_read, params.total_calls);
if (result) { // curl error (connection or transfer failed)
logprintfl (EUCAERROR, "http_put(): %s (%d)\n", error_msg, result);
} else {
long httpcode;
curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &httpcode);
// TODO: pull out response message, too?
switch (httpcode) {
case 200L: // all good
logprintfl (EUCAINFO, "http_put(): file updated sucessfully\n");
code = OK;
break;
case 201L: // all good, created
logprintfl (EUCAINFO, "http_put(): file created sucessfully\n");
code = OK;
break;
case 408L: // timeout, retry
logprintfl (EUCAWARN, "http_put(): server responded with HTTP code %ld (timeout)\n", httpcode);
break;
case 500L: // internal server error (could be a fluke, so we'll retry)
logprintfl (EUCAWARN, "http_put(): server responded with HTTP code %ld (transient?)\n", httpcode);
break;
default: // some kind of error, will not retry
logprintfl (EUCAERROR, "http_put(): server responded with HTTP code %ld\n", httpcode);
retries = 0;
}
}
if (code!=OK && retries > 0) {
logprintfl (EUCAERROR, " upload retry %d of %d will commence in %d seconds\n", TOTAL_RETRIES-retries+1, TOTAL_RETRIES, timeout);
sleep (timeout);
fseek (fp, 0L, SEEK_SET);
timeout <<= 1;
}
retries--;
} while (code!=OK && retries>0);
fclose (fp);
curl_easy_cleanup (curl);
return code;
}
void RAUX::File :: Open ( uint32_t * Status, bool Overwrite )
{
if ( Opened )
{
* Status = kStatus_Failure_FileAlreadyOpen;
return;
}
#if defined ( __APPLE__ )
Handle = fopen ( Name.c_str (), Writable ? ( Overwrite ? "a+b" : "w+b" ) : "rb" );
#elif defined ( _WIN32 )
Handle = fopen ( Name.c_str (), Writable ? ( Overwrite ? "a+b" : "w+b" ) : "rb" );
#else
Handle = fopen64 ( Name.c_str (), Writable ? ( Overwrite ? "a+b" : "w+b" ) : "rb" );
#endif
if ( Handle != NULL )
{
Opened = true;
* Status = kStatus_Success;
}
else
{
switch ( errno )
{
case EINVAL:
* Status = kStatus_Failure_FileModeIncompatible;
break;
case ENOMEM:
* Status = kStatus_Failure_OutOfMemory;
break;
case EACCES:
* Status = kStatus_Failure_Permissions;
break;
case EFAULT:
* Status = kStatus_Failure_InvalidParam;
break;
case EISDIR:
* Status = kStatis_Failure_FileIsDirectory;
break;
default:
* Status = kStatus_Failure_Unknown;
break;
}
}
}
int main(int argc, char** argv) {
char c;
unsigned int i, j, k;
MMPool *mmPool;
dictionary *programInput, *ini;
char argumentText[11] = "argument:0";
double startTime;
double elapsedTime = 0, totalElapsedTime = 0;
BWT *bwt;
HSP *hsp;
unsigned char charMap[256];
unsigned char *convertedKey;
unsigned int *packedKey;
unsigned int found;
unsigned int numberOfPattern, numberOfPatternFound;
unsigned int saIndexLeft, saIndexRight;
SaIndexGroupNew *saIndexGroup;
SaIndexList *tempSaIndex1, *tempSaIndex2;
unsigned int numberOfSaIndexGroup;
HitList *hitList;
unsigned int textPositionMatched, textPositionRetrieved;
unsigned long long totalTextPositionMatched, totalTextPositionRetrieved;
BWTSaRetrievalStatistics BWTSaRetrievalStatistics;
FILE *logFile;
init(textPositionRetrieved); // to avoid compiler warning only
init(textPositionMatched); // to avoid compiler warning only
programInput = ParseInput(argc, argv);
ini = ParseIniFile();
ProcessIni();
ValidateIni();
PrintIni();
if (Confirmation == TRUE) {
printf("BWT Search. Press Y to go or N to cancel. ");
c = (char)getchar();
while (c != 'y' && c != 'Y' && c != 'n' && c!= 'N') {
c = (char)getchar();
}
if (c == 'n' || c == 'N') {
exit(0);
}
}
startTime = setStartTime();
MMMasterInitialize(1, 0, FALSE, NULL);
mmPool = MMPoolCreate(PoolSize);
// Load Database
printf("Loading Database..");
fflush(stdout);
bwt = BWTLoad(mmPool, BWTCodeFileName, BWTOccValueFileName, SaValueFileName, NULL, SaIndexFileName, NULL);
HSPFillCharMap(charMap);
hsp = HSPLoad(mmPool, PackedDNAFileName, AnnotationFileName, AmbiguityFileName, MAX_SEARCH_PATTERN_LENGTH / CHAR_PER_WORD);
if (bwt->textLength != hsp->dnaLength) {
fprintf(stderr, "BWT-BLAST: Database length inconsistent!\n");
exit(1);
}
if (LogFileName[0] != '\0') {
logFile = fopen64(LogFileName, "w");
if (logFile == NULL) {
fprintf(stderr, "Cannot open log file!\n");
exit(1);
}
} else {
logFile = NULL;
}
packedKey = MMPoolDispatch(mmPool, WordPackedLengthFromText(MAX_SEARCH_PATTERN_LENGTH, BIT_PER_CHAR));
convertedKey = MMPoolDispatch(mmPool, MAX_SEARCH_PATTERN_LENGTH);
saIndexGroup = MMUnitAllocate(MaxNumberOfHitGroups * sizeof(SaIndexGroup));
hitList = MMUnitAllocate(MaxNumberOfTextPosition * sizeof(HitList));
tempSaIndex1 = MMUnitAllocate(MaxNumberOfTextPosition * sizeof(SaIndexList));
tempSaIndex2 = MMUnitAllocate(MaxNumberOfTextPosition * sizeof(SaIndexList));
elapsedTime = getElapsedTime(startTime) - totalElapsedTime;
printf("Elapsed time = ");
printElapsedTime(stdout, FALSE, FALSE, TRUE, 4, elapsedTime);
totalElapsedTime += elapsedTime;
printf("\n");
// Process search pattern files
for (i=1; i<=NumberOfSearchPatternFile; i++) {
argumentText[9] = '0' + (char)(i + 2);
iniparser_copystring(programInput, argumentText, PatternFileName, PatternFileName, MAX_FILENAME_LEN);
printf("Loading search pattern : %s\n", PatternFileName);
numberOfPattern = ProcessSearchPattern();
printf("Finished loading search pattern.\n");
elapsedTime = getElapsedTime(startTime) - totalElapsedTime;
printf("Elapsed time = ");
printElapsedTime(stdout, FALSE, FALSE, TRUE, 4, elapsedTime);
totalElapsedTime += elapsedTime;
printf("\n");
if (LogFileName[0] != '\0') {
fprintf(logFile, "Searching for pattern : %s\n", PatternFileName);
}
if (SABinarySearch == TRUE) {
printf("Start forward search with SA index.\n");
if (LogFileName[0] != '\0') {
fprintf(logFile, "Forward search with SA index.\n");
}
numberOfPatternFound = 0;
totalTextPositionMatched = 0;
totalTextPositionRetrieved = 0;
BWTInitializeSaRetrievalStatistics(&BWTSaRetrievalStatistics);
for (j=0; j<numberOfPattern; j++) {
//ConvertTextToWordPacked(searchPattern + searchPatternPosition[j], packedKey, charMap, ALPHABET_SIZE,
// searchPatternPosition[j+1] - searchPatternPosition[j]);
//found = BWTForwardSearchSaIndex(packedKey, searchPatternPosition[j+1] - searchPatternPosition[j],
// bwt, hsp->packedDNA, &saIndexLeft, &saIndexRight);
ConvertTextToCode(searchPattern + searchPatternPosition[j], convertedKey, charMap,
searchPatternPosition[j+1] - searchPatternPosition[j]);
found = BWTSaBinarySearch(convertedKey, searchPatternPosition[j+1] - searchPatternPosition[j],
bwt, hsp->packedDNA, &saIndexLeft, &saIndexRight, packedKey);
if (found) {
numberOfPatternFound++;
textPositionMatched = saIndexRight - saIndexLeft + 1;
totalTextPositionMatched += textPositionMatched;
if (FindTextPosition) {
saIndexGroup->startSaIndex = saIndexLeft;
if (textPositionMatched <= MaxNumberOfTextPosition) {
saIndexGroup->numOfMatch = textPositionMatched;
} else {
saIndexGroup->numOfMatch = MaxNumberOfTextPosition;
printf("Max no of text positions reached! Only %u out of %u text positions retrieved.\n", MaxNumberOfTextPosition, textPositionMatched);
}
saIndexGroup->posQuery = 0;
saIndexGroup->info = 0;
textPositionRetrieved = BWTTextPosition(bwt, saIndexGroup, 1, hitList, tempSaIndex1, tempSaIndex2, &BWTSaRetrievalStatistics, FALSE);
totalTextPositionRetrieved += textPositionRetrieved;
}
}
if (LogFileName[0] != '\0') {
if (found) {
fprintf(logFile, "Pattern number %u found. SA Index from %u to %u. ",
j + 1, saIndexLeft, saIndexRight);
if (FindTextPosition) {
fprintf(logFile, "%u matches of %u located. ", textPositionRetrieved, textPositionMatched);
for (k=0; k<textPositionRetrieved; k++) {
fprintf(logFile, "%u ", hitList[k].posText);
}
}
} else {
fprintf(logFile, "Pattern number %u not found.", j + 1);
}
fprintf(logFile, "\n");
}
}
printf("Finished forward search with SA index.\n");
printf("%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
printf("Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
elapsedTime = getElapsedTime(startTime) - totalElapsedTime;
printf("Elapsed time = ");
printElapsedTime(stdout, FALSE, FALSE, TRUE, 4, elapsedTime);
totalElapsedTime += elapsedTime;
printf("\n");
if (LogFileName[0] != '\0') {
fprintf(logFile, "Finished forward search with SA index.\n");
fprintf(logFile, "%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
fprintf(logFile, "Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
fprintf(logFile, "Elapsed time = ");
printElapsedTime(logFile, FALSE, FALSE, TRUE, 4, elapsedTime);
fprintf(logFile, "\n");
}
}
if (BackwardSearch == TRUE) {
printf("Start backward search.\n");
if (LogFileName[0] != '\0') {
fprintf(logFile, "Backward search.\n");
}
numberOfPatternFound = 0;
totalTextPositionMatched = 0;
totalTextPositionRetrieved = 0;
BWTInitializeSaRetrievalStatistics(&BWTSaRetrievalStatistics);
for (j=0; j<numberOfPattern; j++) {
ConvertTextToCode(searchPattern + searchPatternPosition[j], convertedKey, charMap,
searchPatternPosition[j+1] - searchPatternPosition[j]);
found = BWTBackwardSearch(convertedKey, searchPatternPosition[j+1] - searchPatternPosition[j],
bwt, &saIndexLeft, &saIndexRight);
if (found) {
numberOfPatternFound++;
textPositionMatched = saIndexRight - saIndexLeft + 1;
totalTextPositionMatched += textPositionMatched;
if (FindTextPosition) {
saIndexGroup->startSaIndex = saIndexLeft;
if (textPositionMatched <= MaxNumberOfTextPosition) {
saIndexGroup->numOfMatch = textPositionMatched;
} else {
saIndexGroup->numOfMatch = MaxNumberOfTextPosition;
printf("Max no of text positions reached! Only %u out of %u text positions retrieved.\n", MaxNumberOfTextPosition, textPositionMatched);
}
saIndexGroup->posQuery = 0;
saIndexGroup->info = 0;
textPositionRetrieved = BWTTextPosition(bwt, saIndexGroup, 1, hitList, tempSaIndex1, tempSaIndex2, &BWTSaRetrievalStatistics, FALSE);
totalTextPositionRetrieved += textPositionRetrieved;
}
}
if (LogFileName[0] != '\0') {
if (found) {
fprintf(logFile, "Pattern number %u found. SA Index from %u to %u. Total %u occurrences ", j + 1, saIndexLeft, saIndexRight, saIndexRight - saIndexLeft + 1);
if (FindTextPosition) {
fprintf(logFile, "%u matches of %u located. ", textPositionRetrieved, textPositionMatched);
for (k=0; k<textPositionRetrieved; k++) {
fprintf(logFile, "%u ", hitList[k].posText);
}
}
} else {
fprintf(logFile, "Pattern number %u not found.", j + 1);
}
fprintf(logFile, "\n");
}
}
printf("Finished backward search.\n");
printf("%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
printf("Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
elapsedTime = getElapsedTime(startTime) - totalElapsedTime;
printf("Elapsed time = ");
printElapsedTime(stdout, FALSE, FALSE, TRUE, 4, elapsedTime);
totalElapsedTime += elapsedTime;
printf("\n");
if (LogFileName[0] != '\0') {
fprintf(logFile, "Finished backward search.\n");
fprintf(logFile, "%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
fprintf(logFile, "Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
fprintf(logFile, "Elapsed time = ");
printElapsedTime(logFile, FALSE, FALSE, TRUE, 4, elapsedTime);
fprintf(logFile, "\n");
}
}
if (BackwardSearchCheck == TRUE) {
printf("Start backward search with text.\n");
if (LogFileName[0] != '\0') {
fprintf(logFile, "Backward search with text.\n");
}
numberOfPatternFound = 0;
totalTextPositionMatched = 0;
totalTextPositionRetrieved = 0;
BWTInitializeSaRetrievalStatistics(&BWTSaRetrievalStatistics);
for (j=0; j<numberOfPattern; j++) {
ConvertTextToCode(searchPattern + searchPatternPosition[j], convertedKey, charMap,
searchPatternPosition[j+1] - searchPatternPosition[j]);
ConvertTextToWordPacked(searchPattern + searchPatternPosition[j], packedKey, charMap, ALPHABET_SIZE,
searchPatternPosition[j+1] - searchPatternPosition[j]);
found = BWTBackwardSearchCheckWithText(convertedKey, packedKey, searchPatternPosition[j+1] - searchPatternPosition[j],
bwt, hsp->packedDNA, TextCheckCostFactor, MaxNumberOfTextPosition,
hitList, &saIndexLeft, &saIndexRight);
if (found) {
numberOfPatternFound++;
textPositionMatched = saIndexRight - saIndexLeft + 1;
totalTextPositionMatched += textPositionMatched;
// Find text position if text check not used
if (FindTextPosition && saIndexLeft != 0) {
saIndexGroup->startSaIndex = saIndexLeft;
if (textPositionMatched <= MaxNumberOfTextPosition) {
saIndexGroup->numOfMatch = textPositionMatched;
} else {
saIndexGroup->numOfMatch = MaxNumberOfTextPosition;
printf("Max no of text positions reached! Only %u out of %u text positions retrieved.\n", MaxNumberOfTextPosition, textPositionMatched);
}
saIndexGroup->posQuery = 0;
saIndexGroup->info = 0;
textPositionRetrieved = BWTTextPosition(bwt, saIndexGroup, 1, hitList, tempSaIndex1, tempSaIndex2, &BWTSaRetrievalStatistics, FALSE);
totalTextPositionRetrieved += textPositionRetrieved;
} else {
textPositionRetrieved = textPositionMatched;
totalTextPositionRetrieved += textPositionRetrieved;
}
}
if (LogFileName[0] != '\0') {
if (found) {
fprintf(logFile, "Pattern number %u found. ", j + 1);
if (FindTextPosition) {
fprintf(logFile, "%u matches of %u located. ", textPositionRetrieved, textPositionMatched);
for (k=0; k<textPositionRetrieved; k++) {
fprintf(logFile, "%u ", hitList[k].posText);
}
}
} else {
fprintf(logFile, "Pattern number %u not found.", j + 1);
}
fprintf(logFile, "\n");
}
}
printf("Finished backward search with text.\n");
printf("%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
printf("Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
elapsedTime = getElapsedTime(startTime) - totalElapsedTime;
printf("Elapsed time = ");
printElapsedTime(stdout, FALSE, FALSE, TRUE, 4, elapsedTime);
totalElapsedTime += elapsedTime;
printf("\n");
if (LogFileName[0] != '\0') {
fprintf(logFile, "Finished backward search with text.\n");
fprintf(logFile, "%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
fprintf(logFile, "Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
fprintf(logFile, "Elapsed time = ");
printElapsedTime(logFile, FALSE, FALSE, TRUE, 4, elapsedTime);
fprintf(logFile, "\n");
}
}
if (HammingDistSearch == TRUE) {
printf("Start hamming distance %u approximate match.\n", MaxErrorAllowed);
if (LogFileName[0] != '\0') {
fprintf(logFile, "Hamming distance %u approximate match.\n", MaxErrorAllowed);
}
numberOfPatternFound = 0;
totalTextPositionMatched = 0;
totalTextPositionRetrieved = 0;
BWTInitializeSaRetrievalStatistics(&BWTSaRetrievalStatistics);
for (j=0; j<numberOfPattern; j++) {
ConvertTextToCode(searchPattern + searchPatternPosition[j], convertedKey, charMap,
searchPatternPosition[j+1] - searchPatternPosition[j]);
numberOfSaIndexGroup = BWTHammingDistMatchOld(convertedKey, searchPatternPosition[j+1] - searchPatternPosition[j],
bwt, MaxErrorAllowed, saIndexGroup, MaxNumberOfHitGroups, 0, 0);
if (numberOfSaIndexGroup) {
numberOfPatternFound++;
textPositionMatched = 0;
for (k=0; k<numberOfSaIndexGroup; k++) {
textPositionMatched += saIndexGroup[k].numOfMatch;
}
if (textPositionMatched > MaxNumberOfTextPosition) {
textPositionMatched = 0;
for (k=0; k<numberOfSaIndexGroup && textPositionMatched < MaxNumberOfTextPosition; k++) {
textPositionMatched += saIndexGroup[k].numOfMatch;
}
numberOfSaIndexGroup = k - 1;
saIndexGroup[numberOfSaIndexGroup].numOfMatch -= textPositionMatched - MaxNumberOfTextPosition;
for (; k<numberOfSaIndexGroup; k++) {
textPositionMatched += saIndexGroup[k].numOfMatch;
}
printf("Max no of text positions reached! Only %u out of %u text positions retrieved.\n", MaxNumberOfTextPosition, textPositionMatched);
}
totalTextPositionMatched += textPositionMatched;
if (FindTextPosition) {
textPositionRetrieved = BWTTextPosition(bwt, saIndexGroup, numberOfSaIndexGroup, hitList, tempSaIndex1, tempSaIndex2, &BWTSaRetrievalStatistics, FALSE);
totalTextPositionRetrieved += textPositionRetrieved;
}
}
if (LogFileName[0] != '\0') {
if (numberOfSaIndexGroup) {
fprintf(logFile, "Pattern number %u found. %u matches", j + 1, textPositionMatched);
if (FindTextPosition) {
fprintf(logFile, "%u matches of %u located. ", textPositionRetrieved, textPositionMatched);
for (k=0; k<textPositionRetrieved; k++) {
fprintf(logFile, "%u ", hitList[k].posText);
}
}
} else {
fprintf(logFile, "Pattern number %u not found.", j + 1);
}
fprintf(logFile, "\n");
}
}
printf("Finished hamming distance search.\n");
printf("%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
printf("Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
elapsedTime = getElapsedTime(startTime) - totalElapsedTime;
printf("Elapsed time = ");
printElapsedTime(stdout, FALSE, FALSE, TRUE, 4, elapsedTime);
totalElapsedTime += elapsedTime;
printf("\n");
if (LogFileName[0] != '\0') {
fprintf(logFile, "Finished hamming distance search.\n");
fprintf(logFile, "%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
fprintf(logFile, "Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
fprintf(logFile, "Elapsed time = ");
printElapsedTime(logFile, FALSE, FALSE, TRUE, 4, elapsedTime);
fprintf(logFile, "\n");
}
}
if (EditDistSearch == TRUE) {
printf("Start edit distance %u approximate match.\n", MaxErrorAllowed);
if (LogFileName[0] != '\0') {
fprintf(logFile, "Edit distance %u approximate match.\n", MaxErrorAllowed);
}
numberOfPatternFound = 0;
totalTextPositionMatched = 0;
totalTextPositionRetrieved = 0;
BWTInitializeSaRetrievalStatistics(&BWTSaRetrievalStatistics);
for (j=0; j<numberOfPattern; j++) {
ConvertTextToCode(searchPattern + searchPatternPosition[j], convertedKey, charMap,
searchPatternPosition[j+1] - searchPatternPosition[j]);
numberOfSaIndexGroup = BWTEditDistMatchOld(convertedKey, searchPatternPosition[j+1] - searchPatternPosition[j],
bwt, MaxErrorAllowed, (SaIndexGroupWithLengthError*)saIndexGroup, MaxNumberOfHitGroups);
if (numberOfSaIndexGroup > MaxNumberOfHitGroups) {
fprintf(stderr, "numberOfSaIndexGroup > MaxNumberOfHitGroups!\n");
}
if (numberOfSaIndexGroup) {
numberOfPatternFound++;
textPositionMatched = 0;
for (k=0; k<numberOfSaIndexGroup; k++) {
textPositionMatched += saIndexGroup[k].numOfMatch;
}
if (textPositionMatched > MaxNumberOfTextPosition) {
textPositionMatched = 0;
for (k=0; k<numberOfSaIndexGroup && textPositionMatched < MaxNumberOfTextPosition; k++) {
textPositionMatched += saIndexGroup[k].numOfMatch;
}
numberOfSaIndexGroup = k - 1;
saIndexGroup[numberOfSaIndexGroup].numOfMatch -= textPositionMatched - MaxNumberOfTextPosition;
for (; k<numberOfSaIndexGroup; k++) {
textPositionMatched += saIndexGroup[k].numOfMatch;
}
printf("Max no of text positions reached! Only %u out of %u text positions retrieved.\n", MaxNumberOfTextPosition, textPositionMatched);
}
totalTextPositionMatched += textPositionMatched;
if (FindTextPosition) {
textPositionRetrieved = BWTTextPosition(bwt, saIndexGroup, numberOfSaIndexGroup, hitList, tempSaIndex1, tempSaIndex2, &BWTSaRetrievalStatistics, FALSE);
totalTextPositionRetrieved += textPositionRetrieved;
}
}
if (LogFileName[0] != '\0') {
if (numberOfSaIndexGroup) {
fprintf(logFile, "Pattern number %u found. %u matches. ", j + 1, textPositionMatched);
if (FindTextPosition) {
fprintf(logFile, "%u matches of %u located. ", textPositionRetrieved, textPositionMatched);
for (k=0; k<textPositionRetrieved; k++) {
fprintf(logFile, "%u ", hitList[k].posText);
}
}
} else {
fprintf(logFile, "Pattern number %u not found.", j + 1);
}
fprintf(logFile, "\n");
}
}
printf("Finished edit distance search.\n");
printf("%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
printf("Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
elapsedTime = getElapsedTime(startTime) - totalElapsedTime;
printf("Elapsed time = ");
printElapsedTime(stdout, FALSE, FALSE, TRUE, 4, elapsedTime);
totalElapsedTime += elapsedTime;
printf("\n");
if (LogFileName[0] != '\0') {
fprintf(logFile, "Finished edit distance search.\n");
fprintf(logFile, "%u patterns out of %u found. %llu of %llu matches located.\n", numberOfPatternFound, numberOfPattern, totalTextPositionRetrieved, totalTextPositionMatched);
if (FindTextPosition) {
fprintf(logFile, "Hits: BWT/Cached/Diag/Dup : %u/%u/%u/%u\n",
BWTSaRetrievalStatistics.bwtSaRetrieved, BWTSaRetrievalStatistics.cachedSaRetrieved,
BWTSaRetrievalStatistics.saDiagonalLinked, BWTSaRetrievalStatistics.saDuplicated);
}
fprintf(logFile, "Elapsed time = ");
printElapsedTime(logFile, FALSE, FALSE, TRUE, 4, elapsedTime);
fprintf(logFile, "\n");
}
}
}
MMPoolReturn(mmPool, packedKey, WordPackedLengthFromText(MAX_SEARCH_PATTERN_LENGTH, BIT_PER_CHAR));
MMPoolReturn(mmPool, convertedKey, MAX_SEARCH_PATTERN_LENGTH);
HSPFree(mmPool, hsp, MAX_SEARCH_PATTERN_LENGTH / CHAR_PER_WORD);
BWTFree(mmPool, bwt);
if (searchPattern != NULL) {
MMUnitFree(searchPattern, searchPatternAllocated);
}
if (searchPatternPosition != NULL) {
MMUnitFree(searchPatternPosition, searchPatternPositionAllocated);
}
MMUnitFree(saIndexGroup, MaxNumberOfHitGroups * sizeof(unsigned int));
MMUnitFree(hitList, MaxNumberOfTextPosition * sizeof(unsigned int));
MMPoolFree(mmPool);
iniparser_freedict(programInput);
iniparser_freedict(ini);
return 0;
}
int main(int argc, char *argv[]) {
int fno,c, digit_optind = 0, this_option_optind = optind ? optind : 1, option_index = 0; char *path="";
static struct option long_options[] = { {"r", 0, 0, 'r'}, {0,0, 0, 0} };
for(;;) {
if((c = getopt_long(argc, argv, "xv:", long_options, &option_index)) == -1) break;
switch(c) {
case 0 : printf("Option %s", long_options[option_index].name);
if(optarg) printf (" with arg %s", optarg); printf ("\n"); break;
case 'v': verb = atoi(optarg); break;
default: die("unknown option: %c \n", optopt);
}
}
if(argc - optind < 2) usage();
path = argv[--argc];
for(fno = optind; fno < argc; fno++) {
char outname[257], *inname = argv[fno];
strcpy(outname, path);
char *p = strrchr(inname,'/');
if(!p) p = strrchr(inname,'\\'); if(!p) p=inname;
strcat(outname, p); strcat(outname,".i");
FILE *fi = fopen64(inname, "rb"); if(!fi) { fprintf(stderr, "open error '%s'", inname); perror(inname); exit(-1); } int fdi = fileno(fi);
FILE *fo = fopen64(outname,"wb"),*fm; if(!fo) { fprintf(stderr, "creat error '%s'", outname); perror(outname); exit(-1); } fprintf(stderr, "file='%s'", outname);
fseeko(fo, sizeof(unsigned)+sizeof(unsigned long long), SEEK_SET);
tmap_t *tmap = calloc(1, TERMNUM*sizeof(tmap_t));
if(!tmap) die("malloc error\n");
unsigned *in = NULL,*ip,*ep,num,tid=0,numx=0,outsize;
unsigned char *out = NULL;
unsigned long long fofs; unsigned long long postsize=0,skipsize = 0;
while(fread(&num, 1, 4, fi) == 4) { // read number of docid in term
if(!num) { ++tid; continue; }
unsigned bnum = (num+BLK_DIDNUM-1)/BLK_DIDNUM;
if(num > numx) {
numx = num;
in = realloc(in, num*4+64);
outsize = num*4+bnum*sizeof(unsigned)*SKIP_SIZE+1024;
out = realloc(out, outsize);
if(!in || !out) die("malloc err=%u", num);
}
if(fread(in, 4, num, fi) != num) break; // read docid list
unsigned char *op = out,*_op;
vbput32(op, num); // store f_t
unsigned *pix = (unsigned *)op;
if(num > BLK_DIDNUM) {
op += bnum*sizeof(unsigned)*SKIP_SIZE; skipsize += op-out;
}
for(_op = op, ip = in, ep = ip+num; ip < ep; ) {
unsigned n = min(ep-ip, BLK_DIDNUM), b = 0,bx; if(op+5*n > out+outsize) die("output buffer too small\n");
if(n > 1) {
#ifndef _TURBOPFOR
DELTA(ip, n, b); //bitdenc32( in+1, --n, pa, in[0], mode);
// #ifdef _TURBOPFOR
//b = _p4bits32(ip+1, n-1, &bx);
#endif
}
#ifdef SKIP_S
unsigned u = ip[0]<<SKIP_S|b; // save width
#else
unsigned u = ip[0]; // First docid
#endif
if(num > BLK_DIDNUM) { // skip/index. docid[0] and offset to compressed block
*pix = u;
#if SKIP_SIZE == 2
pix[bnum] = op - _op; // save posting offset
#endif
pix++;
} else vbput32(op, u); // skip not needed
if(n > 1) {
#ifdef _TURBOPFOR
op = n==129?p4d1enc128v32( ip+1, n-1, op, u):p4d1enc32( ip+1, n-1, op, u);//*op++ = bx; op = n==129?_p4enc128v32( ip+1, n-1, op, b, bx):_p4enc32( ip+1, n-1, op, b, bx);
#else
#ifndef SKIP_S
*op++ = b;
#endif
op = n==129?bitpack128v32( ip+1, n-1, op, b) :bitpack32(ip+1, n-1, op, b);
#endif
}
ip += n;
}
fofs = ftello(fo);
tmap_t *t = &tmap[tid++];
TIDMAPSET(t, fofs);
if(fwrite(out, 1, op-out, fo) != op-out) die("fwrite error\n"); postsize += op-out;
}
fofs = ftello(fo); // write termmap
if(fwrite(tmap, sizeof(tmap_t), tid, fo) != tid) die("fwrite error\n");
fseeko(fo, 0, SEEK_SET);
if(fwrite(&fofs, sizeof(unsigned long long), 1, fo) != 1) die("fwrite error\n");
if(fwrite(&tid, sizeof(unsigned), 1, fo) != 1) die("fwrite error\n"); fofs = ftello(fi);
fclose(fi); fclose(fo);
if(in) { free(in); free(out); }
free(tmap);
printf("\nterms=%u size=[tmap=%u skip=%llu post=%llu total=%llu, ratio=%.2f %%\n", tid, (unsigned)(tid*sizeof(tmap_t)),
skipsize, postsize-skipsize, postsize+tid*sizeof(tmap_t)+12, (double)postsize*100/(double)fofs );
}
}
int target_design(ivl_design_t des)
{
ivl_scope_t *roots;
unsigned nroots, idx;
unsigned has_root_scope = 0;
const char*path = ivl_design_flag(des, "-o");
/* Set the indent spacing with the -pspacing flag passed to iverilog
* (e.g. -pspacing=4). The default is 2 spaces. */
const char*spacing_str = ivl_design_flag(des, "spacing");
/* Use -pfileline to determine if file and line information is
* printed for most lines. (e.g. -pfileline=1). The default is no
* file/line information will be printed for individual lines. */
const char*fileline_str = ivl_design_flag(des, "fileline");
/* Use -pallowsigned to allow signed registers/nets and the
* $signed() and $unsigned() system tasks as an extension. */
const char*allowsigned_str = ivl_design_flag(des, "allowsigned");
assert(path);
/* Check for and use a provided indent spacing. */
if (strcmp(spacing_str, "") != 0) {
char *eptr;
long value = strtol(spacing_str, &eptr, 0);
/* Nothing usable in the spacing string. */
if (spacing_str == eptr) {
fprintf(stderr, "vlog95 error: Unable to extract spacing "
"increment from string: %s\n", spacing_str);
return 1;
}
/* Extra stuff at the end. */
if (*eptr != 0) {
fprintf(stderr, "vlog95 error: Extra characters '%s' "
"included at end of spacing string: %s\n",
eptr, spacing_str);
return 1;
}
/* The increment must be greater than zero. */
if (value < 1) {
fprintf(stderr, "vlog95 error: Spacing increment (%ld) must "
"be greater than zero.\n", value);
return 1;
}
/* An increment of more than sixteen is too much. */
if (value > 16) {
fprintf(stderr, "vlog95 error: Spacing increment (%ld) must "
"be sixteen or less.\n", value);
return 1;
}
indent_incr = value;
}
/* Check to see if file/line information should be printed. */
if (strcmp(fileline_str, "") != 0) {
char *eptr;
long value = strtol(fileline_str, &eptr, 0);
/* Nothing usable in the file/line string. */
if (fileline_str == eptr) {
fprintf(stderr, "vlog95 error: Unable to extract file/line "
"information from string: %s\n",
fileline_str);
return 1;
}
/* Extra stuff at the end. */
if (*eptr != 0) {
fprintf(stderr, "vlog95 error: Extra characters '%s' "
"included at end of file/line string: %s\n",
eptr, fileline_str);
return 1;
}
/* The file/line flag must be positive. */
if (value < 0) {
fprintf(stderr, "vlog95 error: File/line flag (%ld) must "
"be positive.\n", value);
return 1;
}
emit_file_line = value > 0;
}
/* Check to see if we should also print signed constructs. */
if (strcmp(allowsigned_str, "") != 0) {
char *eptr;
long value = strtol(allowsigned_str, &eptr, 0);
/* Nothing usable in the allow signed string. */
if (allowsigned_str == eptr) {
fprintf(stderr, "vlog95 error: Unable to extract allow "
"signed information from string: %s\n",
allowsigned_str);
return 1;
}
/* Extra stuff at the end. */
if (*eptr != 0) {
fprintf(stderr, "vlog95 error: Extra characters '%s' "
"included at end of allow signed string: "
"%s\n", eptr, allowsigned_str);
return 1;
}
/* The allow signed flag must be positive. */
if (value < 0) {
fprintf(stderr, "vlog95 error: Allow signed flag (%ld) must "
"be positive.\n", value);
return 1;
}
allow_signed = value > 0;
}
design = des;
#ifdef HAVE_FOPEN64
vlog_out = fopen64(path, "w");
#else
vlog_out = fopen(path, "w");
#endif
if (vlog_out == 0) {
perror(path);
return -1;
}
fprintf(vlog_out, "/*\n");
fprintf(vlog_out, " * 1364-1995 Verilog generated by Icarus Verilog "
"VLOG95 Code Generator,\n");
fprintf(vlog_out, " * Version: " VERSION " (" VERSION_TAG ")\n");
fprintf(vlog_out, " * Converted using %s delays and %s signed support.\n",
ivl_design_delay_sel(des),
allow_signed ? "with" : "without");
fprintf(vlog_out, " */\n");
sim_precision = ivl_design_time_precision(des);
/* Get all the root modules and then convert each one. */
ivl_design_roots(des, &roots, &nroots);
/* Emit any root scope tasks or functions first. */
for (idx = 0; idx < nroots; idx += 1) {
switch(ivl_scope_type(roots[idx])) {
case IVL_SCT_FUNCTION:
case IVL_SCT_TASK:
if (! has_root_scope) {
fprintf(vlog_out, "module ivl_root_scope;\n");
indent += indent_incr;
has_root_scope = 1;
}
/* Say this task/function has a parent so the
* definition is emitted correctly. */
emit_scope(roots[idx], roots[idx]);
break;
default:
break;
}
}
if (has_root_scope) {
indent -= indent_incr;
assert(indent == 0);
fprintf(vlog_out, "endmodule /* ivl_root_scope */\n");
}
/* Emit the rest of the scope objets. */
for (idx = 0; idx < nroots; idx += 1) emit_scope(roots[idx], 0);
free_emitted_scope_list();
/* Emit any UDP definitions that the design used. */
emit_udp_list();
/* Emit any UDPs that are Icarus generated (D-FF). */
emit_icarus_generated_udps();
/* If there were errors then add this information to the output. */
if (vlog_errors) {
fprintf(vlog_out, "\n");
fprintf(vlog_out, "/*\n");
if (vlog_errors == 1) {
fprintf(vlog_out, " * There was 1 error during "
"translation.\n");
} else {
fprintf(vlog_out, " * There were %d errors during "
"translation.\n",
vlog_errors);
}
fprintf(vlog_out, " */\n");
/* Add something that makes the file invalid to make sure
* the user knows there were errors. */
fprintf(vlog_out, "<Add some text to make sure this file is not "
"valid Verilog>\n");
}
fclose(vlog_out);
/* A do nothing call to prevent warnings about this routine not
* being used. */
dump_nexus_information(0, 0);
return vlog_errors;
}
HTAAProt *HTAAProt_new( char *cur_docname, char *prot_filename, char *ids )
{
int eax;
HTList *cur = prot_cache;
HTAAProtCache *cache_item = 0;
HTAAProt *prot;
FILE *fp;
if ( prot_cache == 0 )
{
prot_cache = HTList_new( );
do
{
if ( cur )
{
cur = &cur->next;
cache_item = 0;
}
}
while ( cache_item && strcmp( &cache_item->prot_filename[0], prot_filename ) );
if ( cache_item )
{
prot = &cache_item->prot;
if ( WWW_TraceFlag )
{
fprintf( TraceFP( ), "%s `%s' already in cache\n", "HTAAProt_new: Protection file", prot_filename );
}
}
else
{
if ( WWW_TraceFlag )
{
fprintf( TraceFP( ), "HTAAProt_new: Loading protection file `%s'\n", prot_filename );
}
prot = calloc( 1, sizeof( HTAAProt ) );
if ( prot == 0 )
outofmem( "../../../WWW/Library/Implementation/HTAAProt.c", "HTAAProt_new" );
*(int*)&prot->ctemplate[0] = 0;
*(int*)&prot->filename = 0;
*(int*)&prot->uid_name = 0;
*(int*)&prot->gid_name = 0;
prot->valid_schemes = HTList_new( );
*(int*)&prot->mask_group = 0;
prot->values = HTAssocList_new( );
if ( prot_filename )
{
fp = fopen64( prot_filename, "r" );
if ( fp )
{
HTAA_parseProtFile( prot, fp );
fclose( fp );
cache_item = calloc( 1, sizeof( HTAAProtCache ) );
if ( cache_item == 0 )
outofmem( "../../../WWW/Library/Implementation/HTAAProt.c", "HTAAProt_new" );
cache_item->prot = prot;
*(int*)&cache_item->prot_filename[0] = 0;
HTSACopy( &cache_item->prot_filename, prot_filename );
HTList_addObject( prot_cache, (void*)cache_item );
}
}
if ( WWW_TraceFlag )
{
fprintf( TraceFP( ), "HTAAProt_new: %s `%s'\n", "Unable to open protection setup file", "(null)" );
}
}
if ( cur_docname )
HTSACopy( &prot->filename, cur_docname );
HTAA_setIds( prot, ids );
return prot;
}
}
NV_INT32 read_srtm1_topo_one_degree (NV_INT32 lat, NV_INT32 lon, NV_INT16 **array)
{
static NV_CHAR dir[512], file[512], version[128], zversion[128];
static NV_INT32 header_size, prev_lat = -999, prev_lon = -999, prev_block = -1;
FILE *block_fp;
NV_CHAR varin[1024], info[1024], header_block[HEADER_SIZE];
NV_U_BYTE *buf, *bit_box = NULL, head[4];
NV_INT32 i, j, shift_lat, shift_lon, resolution, pos, size = 0, status, ndx, block;
uLong csize;
uLongf bsize;
NV_INT64 address;
NV_INT16 start_val, bias, null_val, num_bits, temp, last_val;
if (no_file) return (-1);
/* First time through, open the file and read the header. */
if (first)
{
if (getenv ("SRTM_DATA") == NULL)
{
fprintf (stderr, "Unable to find SRTM_DATA environment variable\n");
fflush (stderr);
no_file = NVTrue;
return (-1);
}
strcpy (dir, getenv ("SRTM_DATA"));
sprintf (file, "%s%1csrtm1%1csrtm1_block_map.dat", dir, SEPARATOR, SEPARATOR);
if ((block_fp = fopen (file, "rb")) == NULL)
{
perror (file);
no_file = NVTrue;
return (-1);
}
fseek (block_fp, 0, SEEK_SET);
fread (block_map, sizeof (block_map), 1, block_fp);
fclose (block_fp);
first = NVFalse;
}
/* If we're working in the real 0-360 world (where 0 to 0 is 0 to 360) we want to turn longitudes greater
than 180 into negatives before we switch to the bogus 0-360 world (where -180 to 180 is 0 to 180). */
if (lon >= 180) lon -= 360;
/* Shift into a 0 to 180 by 0 to 360 world. */
shift_lat = lat + 90;
shift_lon = lon + 180;
block = shift_lat * 360 + shift_lon;
if (block_map[block] != prev_block)
{
/* If the block_map value is 0 then no data was loaded for this cell. */
if (!block_map[block]) return (2);
prev_block = block;
sprintf (file, "%s%1csrtm1%1cRegion_0%1d.cte", dir, SEPARATOR, SEPARATOR, block_map[block]);
/* Note that we are returning undefined if we can't find the file. This is to allow the use of
a subset of the area block files without requiring all of the files to be present. */
if (fp) fclose (fp);
if ((fp = fopen64 (file, "rb")) == NULL) return (2);
/* Allocate the map memory. */
if (map) free (map);
map = (NV_U_BYTE *) calloc (MAP_BYTES, sizeof (NV_U_BYTE));
if (map == NULL)
{
perror ("Allocating map memory");
exit (-1);
}
/* Read the header block. */
fread (header_block, HEADER_SIZE, 1, fp);
ndx = 0;
while (header_block[ndx] != 0)
{
for (i = 0 ; i < 1024 ; i++)
{
if (header_block[ndx] == '\n') break;
varin[i] = header_block[ndx];
ndx++;
}
varin[i] = 0;
if (strstr (varin, "[END OF HEADER]")) break;
/* Put everything to the right of the equals sign in 'info'. */
if (strchr (varin, '=') != NULL) strcpy (info, (strchr (varin, '=') + 1));
if (strstr (varin, "[VERSION]")) strcpy (version, info);
if (strstr (varin, "[ZLIB VERSION]"))
{
strcpy (zversion, info);
sscanf (zversion, "%d.", &i);
sscanf (zlibVersion (), "%d.", &j);
if (i != j)
{
fprintf (stderr, "\n\nZlib library version (%s) is not compatible with version used to build SRTM file (%s)\n\n",
zlibVersion (), zversion);
exit (-1);
}
}
if (strstr (varin, "[HEADER SIZE]")) sscanf (info, "%d", &header_size);
ndx++;
}
if (header_size != HEADER_SIZE)
{
fprintf (stderr, "Header sizes do not match, WTF, over!\n");
exit (-1);
}
/* Move past the end of the header and read the map. */
fseeko64 (fp, (NV_INT64) header_size, SEEK_SET);
fread (map, MAP_BYTES, 1, fp);
first = NVFalse;
}
/* Only read the data if we have changed one-degree cells. */
if (prev_lat != shift_lat || prev_lon != shift_lon)
{
/* Unpack the address from the map. */
address = srtm_double_bit_unpack (map, (shift_lat * 360 + shift_lon) * 36, 36);
prev_lat = shift_lat;
prev_lon = shift_lon;
/* If the address is 0 (water) or 2 (undefined), return the address. */
if (address < header_size) return ((NV_INT32) address);
/* Move to the address and read/unpack the header. */
fseek (fp, address, SEEK_SET);
fread (head, 8, 1, fp);
pos = 0;
resolution = (NV_INT32) srtm_bit_unpack (head, pos, 3); pos += 3;
csize = (uLong) srtm_bit_unpack (head, pos, 30); pos += 30;
bsize = (uLongf) srtm_bit_unpack (head, pos, 31);
size = 3600;
/* We have to set an approximate size for unpacking (see the ZLIB documentation). */
bsize += NINT ((NV_FLOAT32) bsize * 0.10) + 12;
/* Allocate the uncompressed memory. */
bit_box = (NV_U_BYTE *) calloc (bsize, sizeof (NV_U_BYTE));
if (bit_box == NULL)
{
perror ("Allocating bit_box memory in read_srtm1_topo");
exit (-1);
}
/* Allocate the compressed memory. */
buf = (NV_U_BYTE *) calloc (csize, sizeof (NV_U_BYTE));
if (buf == NULL)
{
perror ("Allocating buf memory");
exit (-1);
}
/* Read the compressed data. */
fread (buf, csize, 1, fp);
/* Uncompress the data. */
status = uncompress (bit_box, &bsize, buf, csize);
if (status)
{
fprintf (stderr, "Error %d uncompressing record\n", status);
exit (-1);
}
free (buf);
/* Unpack the internal header. */
pos = 0;
start_val = srtm_bit_unpack (bit_box, pos, 16); pos += 16;
bias = srtm_bit_unpack (bit_box, pos, 16); pos += 16;
num_bits = srtm_bit_unpack (bit_box, pos, 4); pos += 4;
null_val = NINT (pow (2.0L, (NV_FLOAT64) num_bits)) - 1;
/* Allocate the cell memory. */
if (box != NULL) free (box);
box = (NV_INT16 *) calloc (size * size, sizeof (NV_INT16));
if (box == NULL)
{
perror ("Allocating box memory in read_srtm1_topo");
exit (-1);
}
/* Uncompress the data (delta coded snake dance). */
last_val = start_val;
for (i = 0 ; i < size ; i++)
{
if (!(i % 2))
{
for (j = 0 ; j < size ; j++)
{
temp = srtm_bit_unpack (bit_box, pos, num_bits); pos += num_bits;
if (temp < null_val)
{
box[i * size + j] = last_val + temp - bias;
last_val = box[i * size + j];
}
else
{
box[i * size + j] = -32768;
}
}
}
else
{
for (j = size - 1 ; j >= 0 ; j--)
{
temp = srtm_bit_unpack (bit_box, pos, num_bits); pos += num_bits;
if (temp < null_val)
{
box[i * size + j] = last_val + temp - bias;
last_val = box[i * size + j];
}
else
{
box[i * size + j] = -32768;
}
}
}
}
free (bit_box);
prev_size = size;
}
/* We didn't change cells so set the same size as last time. */
else
{
size = prev_size;
}
*array = box;
return (size);
}
/**
* @fn int z_compress(z_t zip, const z_file_t zname, const char* password, z_clevel_et level, _Bool append, _Bool exclude_path, fifo_t files)
* @brief Creation of a new ZIP file.
* @param zip The ZIP context.
* @param zname The zip file name.
* @param password the zip password else NULL or empty.
* @param level The compression level.
* @param append Append mode.
* @param exclude_path Exclude the file path.
* @param files The file list.
* @retunr 0 on success else -1.
*/
int z_compress(z_t zip, const z_file_t zname, const char* password, z_clevel_et level, _Bool append, _Bool exclude_path, fifo_t files) {
struct z_s* z = Z_CAST(zip);
z_file_t filename_try;
int size_buf = 0;
void* buf = NULL;
zipFile zf;
size_buf = Z_WRITE_BUFFER_SIZE;
buf = (void*)malloc(size_buf);
if (!buf) {
logger(LOG_ERR, "Error allocating memory\n");
return -1;
}
bzero(filename_try, sizeof(z_file_t));
strcpy(filename_try, zname);
if(!string_indexof(filename_try, ".") == -1)
strcat(filename_try, ".zip");
zf = zipOpen64(filename_try, (append) ? 2 : 0);
if (!zf) {
free(buf);
logger(LOG_ERR, "Error opening %s\n", filename_try);
return -1;
} else
logger(LOG_DEBUG, "Creating %s\n", filename_try);
while(!fifo_empty(files)) {
const char* filenameinzip = fifo_pop(files);
FILE * fin;
int size_read;
const char *savefilenameinzip;
zip_fileinfo zi;
unsigned long crc_file = 0;
int zip64 = 0;
memset(&zi, 0, sizeof(zip_fileinfo));
if(file_is_dir(filenameinzip)) {
((char*)filenameinzip)[strlen(filenameinzip)] = z->dir_delimiter;
strncat((char*)filenameinzip, ".empty", sizeof(file_name_t));
file_touch(filenameinzip);
}
logger(LOG_DEBUG, "Trying to add file '%s'\n", filenameinzip);
file_time(filenameinzip, (struct tm*)&zi.tmz_date);
if(password != NULL && strlen(password))
if(z_get_file_crc(filenameinzip, buf, size_buf, &crc_file) != ZIP_OK) {
zipClose(zf, NULL);
free(buf);
logger(LOG_ERR, "Error getting the crc for the file %s\n", filenameinzip);
return -1;
}
zip64 = file_is_large_file(filenameinzip);
/* The path name saved, should not include a leading slash. */
/*if it did, windows/xp and dynazip couldn't read the zip file. */
savefilenameinzip = filenameinzip;
while(savefilenameinzip[0] == z->dir_delimiter)
savefilenameinzip++;
/*should the zip file contain any path at all?*/
if(exclude_path) {
const char *tmpptr;
const char *lastslash = 0;
for(tmpptr = savefilenameinzip; *tmpptr; tmpptr++) {
if(*tmpptr == z->dir_delimiter)
lastslash = tmpptr;
}
if(lastslash)
savefilenameinzip = lastslash+1; // base filename follows last slash.
}
if(zipOpenNewFileInZip3_64(zf, savefilenameinzip, &zi,
NULL, 0, NULL, 0, NULL /* comment*/,
(level != 0) ? Z_DEFLATED : 0, level,0,
-MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
(password != NULL && strlen(password)) ? password : NULL, crc_file, zip64) != ZIP_OK) {
zipClose(zf, NULL);
free(buf);
logger(LOG_ERR, "Error in opening %s in zipfile\n", filenameinzip);
return -1;
}
fin = fopen64(filenameinzip, "rb");
if(!fin) {
zipCloseFileInZip(zf);
zipClose(zf, NULL);
free(buf);
logger(LOG_ERR, "Error in opening %s for reading\n", filenameinzip);
return -1;
}
do {
size_read = (int)fread(buf,1,size_buf,fin);
if(size_read < size_buf)
if(!feof(fin)) {
logger(LOG_ERR, "Error in reading %s\n",filenameinzip);
break;
}
if (size_read > 0) {
if(zipWriteInFileInZip(zf, buf, size_read) < 0) {
logger(LOG_ERR, "Error in writing %s in the zipfile\n", filenameinzip);
break;
}
}
} while(size_read > 0);
if(fin) fclose(fin);
if(zipCloseFileInZip(zf) != ZIP_OK) {
logger(LOG_ERR, "Error in closing %s in the zipfile\n", filenameinzip);
break;
}
}
if(zipClose(zf, NULL) != ZIP_OK)
logger(LOG_ERR, "Error in closing %s\n",filename_try);
free(buf);
return 0;
}
// ////////////////////////////////////////////////////////////////////////////////
// @global 释放单个文件
//
int ExtractSingleFile(unzFile& uf)
{
char filename_inzip[256] = {0};
unz_file_info64 file_info;
int err = unzGetCurrentFileInfo64(uf,&file_info,filename_inzip,sizeof(filename_inzip),NULL,0,NULL,0);
if ( UNZ_OK != err )
{
DebugTools::OutputDebugPrintfW(
L"[ZipTools] [ExtractSingleFile] unzGetCurrentFileInfo64 Failed.[%d]\r\n", err);
return err;
}
unsigned int size_buf = WRITEBUFFERSIZE;
PBYTE buf = new BYTE[size_buf];
// 判断名字是否以 '\\' 结尾，如果以 '\\'
// 结尾，代表这是一个目录名，创建目录，
// 反之则创建文件
if ( '\\' == filename_inzip[strlen(filename_inzip)-1] )
{
WCHAR wzDirName[1024] = {0};
MultiByteToWideChar(CP_ACP, 0, filename_inzip, -1, wzDirName, 1024);
FileTools::CreateDirectorys(wzDirName);
}
else
{
err = unzOpenCurrentFilePassword(uf, NULL);
if ( err != UNZ_OK)
{
DebugTools::OutputDebugPrintfW(
L"[ZipTools] [ExtractSingleFile] unzOpenCurrentFilePassword Failed.[%d]\r\n", err);
}
// 创建文件所在的目录
WCHAR wzFileName[MAX_PATH] = {0};
MultiByteToWideChar(CP_ACP, 0, filename_inzip, -1, wzFileName, MAX_PATH);
WCHAR wzDirName[MAX_PATH] = {0};
FileTools::GetFileDir(wzFileName, wzDirName, L'\\');
FileTools::CreateDirectorys(wzDirName);
DebugTools::OutputDebugPrintfW(
L"[FileTools] [UnZip] Extracting %s ...\r\n", wzFileName);
FILE* fout = fopen64(filename_inzip, "wb");
do
{
err = unzReadCurrentFile(uf,buf,size_buf);
if ( err < 0 )
{
DebugTools::OutputDebugPrintfW(
L"[ZipTools] [ExtractSingleFile] unzReadCurrentFile Failed.[%d]\r\n", err);
break;
}
if ( err > 0 )
{
if ( fwrite(buf,err,1,fout) != 1 )
{
DebugTools::OutputDebugPrintfW(
L"[ZipTools] [ExtractSingleFile] Write File Failed.[%s]\r\n",
filename_inzip);
err = UNZ_ERRNO;
break;
}
}
} while ( err > 0 );
if ( fout )
{
fclose(fout);
}
// 修改文件时间
ChangeFileTime(filename_inzip, file_info.dosDate, file_info.tmu_date);
unzCloseCurrentFile(uf);
delete [] buf;
}
}
/***
GenerateROWfile
Creates a .ROW file containing in which nodes were running (if some input has NODE info).
***/
int GenerateROWfile (char *name, struct Pair_NodeCPU *info, int nfiles, struct input_t *files)
{
int i, j, k;
int numNodes;
int numCPUs;
char FORMAT[128];
FILE *fd;
/* Compute how many CPUs and NODEs */
numNodes = numCPUs = 0;
while (info[numNodes].CPUs > 0)
{
numCPUs += info[numNodes].CPUs;
numNodes ++;
}
/* This will provide %04d.%s pex */
sprintf (FORMAT, "%%0%dd.%%s", FLOG(numCPUs));
#if HAVE_FOPEN64
fd = fopen64 (name, "w");
#else
fd = fopen (name, "w");
#endif
fprintf (fd, "LEVEL CPU SIZE %d\n", numCPUs);
/* K will be our "Global CPU" counter */
k = 1;
for (i = 0; i < numNodes; i++)
{
char *node = info[i].files[0]->node;
for (j = 0; j < info[i].CPUs; j++)
{
fprintf (fd, FORMAT, k, node);
fprintf (fd, "\n");
k++;
}
}
fprintf (fd, "\nLEVEL NODE SIZE %d\n", numNodes);
for (i = 0; i < numNodes; i++)
fprintf (fd, "%s\n", info[i].files[0]->node);
if (!get_option_merge_NanosTaskView())
{
qsort (files, nfiles, sizeof(input_t), SortByObject);
fprintf (fd, "\nLEVEL THREAD SIZE %d\n", numCPUs);
for (i = 0; i < nfiles; i++)
fprintf (fd, "%s\n", files[i].threadname);
qsort (files, nfiles, sizeof(input_t), SortByOrder);
}
else
{
/* What naming scheme should we follow in Nanos Task View?
While undecided, keep this clear. Paraver will handle it. */
}
fclose (fd);
return 0;
}
/**********************************************
RecurseFiles(): Process all files in all directories.
**********************************************/
void RecurseFiles (char *S)
{
char NewS[FILENAME_MAX+1];
DIR *Dir;
struct dirent *Entry;
struct stat64 Stat;
CksumFile *CF;
char *Result=NULL;
Cksum *Sum;
Dir = opendir(S);
if (Dir == NULL)
{
Result=NULL;
/* it's a single file -- compute checksum */
CF = SumOpenFile(S);
if (CF == NULL)
{
FILE *Fin;
Fin = fopen64(S,"rb");
if (!Fin)
{
perror("Huh?");
fprintf(stderr,"ERROR: cannot open file \"%s\".\n",S);
}
else
{
Sum = SumComputeFile(Fin);
if (Sum) { Result=SumToString(Sum); free(Sum); }
fclose(Fin);
}
}
else
{
Sum = SumComputeBuff(CF);
if (Sum) { Result=SumToString(Sum); free(Sum); }
SumCloseFile(CF);
}
if (Result != NULL)
{
printf("%s %s\n",Result,S);
free(Result);
Result=NULL;
}
return;
}
Entry = readdir(Dir);
while(Entry != NULL)
{
if (!strcmp(Entry->d_name,".")) goto skip;
if (!strcmp(Entry->d_name,"..")) goto skip;
memset(NewS,'\0',sizeof(NewS));
strcpy(NewS,S);
strcat(NewS,"/");
strcat(NewS,Entry->d_name);
lstat64(NewS,&Stat);
Result=NULL;
if (S_ISDIR(Stat.st_mode)) RecurseFiles(NewS);
else
{
/* compute checksum */
CF = SumOpenFile(NewS);
if (CF == NULL)
{
FILE *Fin;
Fin = fopen64(NewS,"rb");
if (!Fin)
fprintf(stderr,"ERROR: Cannot open file \"%s\".\n",NewS);
else
{
Sum = SumComputeFile(Fin);
if (Sum) { Result=SumToString(Sum); free(Sum); }
fclose(Fin);
}
}
else
{
Sum = SumComputeBuff(CF);
if (Sum) { Result=SumToString(Sum); free(Sum); }
SumCloseFile(CF);
}
if (Result != NULL)
{
printf("%s %s\n",Result,NewS);
free(Result);
Result=NULL;
}
}
skip:
Entry = readdir(Dir);
}
closedir(Dir);
} /* RecurseFiles() */
int main(int argc, char **argv)
{
int pass = 0;
int bNew = 0;
long batch = -1;
long idxStart = 0;
long idxEnd = 0;
long idx = 1;
long seq = 0;
long titlesToProcess = 0;
char sFileName[1024];
char sServer[256];
char sDB[256];
char sId[256];
char sPassword[256];
char sTitle[256];
off64_t file_offset_for_pass_1;
long max_article_idx;
MYSQL *conn;
MYSQL *conn2;
MYSQL_RES *res;
MYSQL_ROW row;
char sSQL[MAX_SQL_STR];
int rc;
char c;
time_t t;
nMsgLevel = 0;
printf("wiki-xml-parser - (C) 2009 by Openmoko Inc.\n"
"This program is Free Software and has ABSOLUTELY NO WARRANTY\n\n");
if (argc <2)
{
help();
exit(2);
}
sFileName[0] = '\0';
sServer[0] = '\0';
sDB[0] = '\0';
sId[0] = '\0';
sPassword[0] = '\0';
sTitle[0] = '\0';
while (1) {
int c, option_index = 0;
c = getopt_long(argc, argv, "hf:p:nb:g:e:m:t:u:w:s:d:i:l:q:", opts,
&option_index);
if (c == -1)
break;
switch (c) {
case 'h':
help();
exit(0);
break;
case 'f':
strncpy(sFileName, optarg, sizeof(sFileName) - 1);
sFileName[sizeof(sFileName) - 1] = '\0';
break;
case 'p':
pass = atoi(optarg);
break;
case 'b':
batch = atoi(optarg);
break;
case 'g':
idxStart = atol(optarg);
break;
case 'e':
idxEnd = atol(optarg);
break;
case 'n':
bNew = 1;
break;
case 'm':
nMsgLevel = atoi(optarg);
break;
case 't':
titlesToProcess = atol(optarg);
break;
case 'u':
strncpy(sId, optarg, sizeof(sId) - 1);
sId[sizeof(sId) - 1] = '\0';
break;
case 'w':
strncpy(sPassword, optarg, sizeof(sPassword) - 1);
sPassword[sizeof(sPassword) - 1] = '\0';
break;
case 's':
strncpy(sServer, optarg, sizeof(sServer) - 1);
sServer[sizeof(sServer) - 1] = '\0';
break;
case 'd':
strncpy(sDB, optarg, sizeof(sDB) - 1);
sDB[sizeof(sDB) - 1] = '\0';
break;
case 'i':
idx = atol(optarg);
break;
case 'q':
seq = atol(optarg);
break;
case 'l':
strncpy(sTitle, optarg, sizeof(sTitle) - 1);
sServer[sizeof(sTitle) - 1] = '\0';
break;
default:
help();
exit(2);
}
}
if (pass == 5)
{
generate_pedia_hsh();
exit(0);
}
if (!titlesToProcess)
titlesToProcess = 1024;
if (!sFileName[0])
strcpy(sFileName, "enwiki-latest-pages-articles.xml");
if (!sId[0])
strcpy(sId, "root");
if (!sServer[0])
strcpy(sServer, "localhost");
if (!sDB[0])
strcpy(sDB, "wikixml");
if (batch >= 0)
{
idxStart = MAX_ARTICLES_PER_DAT * batch;
idxEnd = idxStart + MAX_ARTICLES_PER_DAT - 1;
if (!idxStart)
idxStart = 1;
}
else if (idxEnd < idxStart)
idxEnd = idxStart;
my_init();
g_conn = mysql_init(NULL);
if (!mysql_real_connect(g_conn, sServer, sId, sPassword, sDB, 0, NULL, 0))
{
showMsg(0, "Error connecting DB %s/%s using %s/%s\n", sServer, sDB, sId, sPassword);
exit(1);
}
conn = mysql_init(NULL);
if (!mysql_real_connect(conn, sServer, sId, sPassword, sDB, 0, NULL, 0))
{
showMsg(0, "Error connecting DB %s/%s using %s/%s\n", sServer, sDB, sId, sPassword);
exit(1);
}
if (pass==0)
{
if (sTitle[0])
sprintf(sSQL, "select title, text_start_offset, text_len, redirect_title, entry_type from entries where title='%s'", sTitle);
else if (seq)
sprintf(sSQL, "select title, text_start_offset, text_len, redirect_title, entry_type from entries where seq=%ld", seq);
else
sprintf(sSQL, "select title, text_start_offset, text_len, redirect_title, entry_type from entries where idx=%ld", idx);
printf("SQL: %s\n", sSQL);
rc = mysql_query(conn, sSQL);
if (rc)
{
showMsg(0, "query entries error - %d (%s)\n", rc, mysql_error(conn));
exit(1);
}
res = mysql_use_result(conn);
if ((row = mysql_fetch_row(res)) != NULL)
{
FILE *fd;
off64_t nArticleOffset;
long nTextLen;
char buf[1025];
int len;
if (!row[1])
{
if (row[4])
printf("title: [%s], redirect: [%s]\n", row[0], row[3]);
else
printf("title: [%s], entry type: %s\n", row[0], row[4]);
exit(0);
}
sscanf(row[1], "%Ld", &nArticleOffset);
nTextLen = atol(row[2]);
printf("offset: %Lx, len: %lx\n\n", nArticleOffset, nTextLen);
fd = fopen64(sFileName, "rb");
fseeko64(fd, nArticleOffset, SEEK_SET);
printf("<mediawiki>\n<page>\n<title>%s</title>\n <revision>\n<text>", row[0]);
while (nTextLen > 0)
{
if (nTextLen > 1024)
len = fread(buf, 1, 1024, fd);
else
len = fread(buf, 1, nTextLen, fd);
nTextLen -= 1024;
buf[len] = '\0';
printf(buf);
}
printf("\n</text>\n</revision>\n</page>\n</mediawiki>");
}
else
printf("not found\n");
exit(0);
}
else if (pass == 4)
{
long idx_range[MAX_IDX_RANGE][2];
long idx_range_count = 0;
long count = 0;
int i;
long max_file_size = 0;
count_files("./dat", &idx_range_count, idx_range, &max_file_size);
for (i = 0; i < idx_range_count; i++)
{
showMsg(0, "%8ld ~ %8ld\n", idx_range[i][0], idx_range[i][1]);
count += idx_range[i][1] - idx_range[i][0] + 1;
}
showMsg(0, "Total file count: %ld, Max file size: %ld\n", count, max_file_size);
exit(0);
}
rc = mysql_query(conn, "select idx from entries order by idx desc limit 1");
res = mysql_use_result(conn);
if ((row = mysql_fetch_row(res)) != NULL && row[0])
{
showMsg(0, "max article idx:\t\t%ld\n", (max_article_idx=atol(row[0])));
mysql_free_result(res);
mysql_commit(conn);
rc = mysql_query(conn, "select count(*) from entries");
res = mysql_use_result(conn);
if ((row = mysql_fetch_row(res)) != NULL && row[0])
showMsg(0, "pass1 processed entries:\t%ld\n", atol(row[0]));
// mysql_free_result(res);
// mysql_commit(conn);
// rc = mysql_query(conn, "select count(*) from entries where entry_type=0 and pass_2_processed=1");
// res = mysql_use_result(conn);
// if ((row = mysql_fetch_row(res)) != NULL && row[0])
// showMsg(0, "pass2 processed articles:\t%ld\n", atol(row[0]));
// mysql_free_result(res);
// mysql_commit(conn);
// rc = mysql_query(conn, "select count(*) from entries where entry_type=0 and pass_2_processed=0");
// res = mysql_use_result(conn);
// if ((row = mysql_fetch_row(res)) != NULL && row[0])
// showMsg(0, "pass2 to be processed articles:\t%ld\n", atol(row[0]));
}
else
{
showMsg(0, "no process status\n");
file_offset_for_pass_1 = 0;
max_article_idx = 0;
init_bigram_table(conn);
}
mysql_free_result(res);
if (pass == 1)
{
// if (nBatch <= 0)
// nBatch = 8;
// else if (nBatch > MAX_DAT_FILES)
// nBatch = MAX_DAT_FILES;
showMsg(0, "\nProcessing %s for %ld titles.\n", sFileName, titlesToProcess);
}
else if (pass == 2)
showMsg(0, "\nProcessing idx %ld ~ %ld.\n", idxStart, idxEnd);
showMsg(3, "Server-%s, DB-%s, user-%s, password-%s.\n\n", sServer, sDB, sId, sPassword);
if (bNew && pass == 1)
printf("\nAll previously processed records will be deleted. Continue processing? (y to continue)\n");
else
printf("\nStart processing? (y to continue)\n");
c = getchar();
if (c == 'y')
{
if (bNew)
{
if (pass == 1)
{
mysql_query(conn, "delete from entries");
rc = mysql_commit(conn);
mysql_query(conn, "alter table entries AUTO_INCREMENT=1");
rc = mysql_commit(conn);
showMsg(3, "commit update process_status rc=%d\n", rc);
file_offset_for_pass_1 = 0;
max_article_idx = 0;
init_bigram_table(conn);
}
}
time(&t);
showMsg(0, "start pass %d - %s\n", pass, ctime(&t));
if (pass == 1)
{
process_pass_1(conn, sFileName, nMsgLevel, titlesToProcess,
file_offset_for_pass_1, max_article_idx);
}
else if (pass == 2)
{
conn2 = mysql_init(NULL);
if (!mysql_real_connect(conn2, sServer, sId, sPassword, sDB, 0, NULL, 0))
{
showMsg(0, "Error connecting DB %s/%s using %s/%s\n", sServer, sDB, sId, sPassword);
exit(1);
}
process_pass_2(conn, conn2, sFileName, nMsgLevel, titlesToProcess, batch, idxStart, idxEnd);
mysql_close(conn2);
}
else if (pass == 3)
{
conn2 = mysql_init(NULL);
if (!mysql_real_connect(conn2, sServer, sId, sPassword, sDB, 0, NULL, 0))
{
showMsg(0, "Error connecting DB %s/%s using %s/%s\n", sServer, sDB, sId, sPassword);
exit(1);
}
unlink("pedia.idx");
unlink("pedia.fnd");
unlink("pedia.pfx");
if (max_article_idx > 2000)
process_pass_3(conn, conn2, 1);
else
process_pass_3(conn, conn2, 0);
mysql_close(conn2);
}
time(&t);
showMsg(0, "end pass %d - %s\n", pass, ctime(&t));
}
/* close connection */
mysql_close(conn);
mysql_close(g_conn);
exit(0);
}
/*
* Reads a file into memory and builds a linked-list containing all the sequences found in the file.
* Uses stat64() and fopen64() to allow for loading files larger than 2GB.
*
* All file data is stored in the fileBuffer array and each of the sequence structs use pointers to
* access their respective data.
*
* ARGS
* char* inputFile - String containing the file location.
* Sequence* currentSequence - pointer to the memory location of the first member of a linked list
* that will contain all the sequences read in from the file.
*
* RETURN: char* - returns a pointer to an array containing all data read in from the file. The sequence
linked list contains pointers to their respective information.
*/
char* read_file(char* inputFile, Sequence* currentSequence, ScoringMatrix* score, int* numberOfSequences)
{
char* fileBuffer;
int fileSize;
int c;
int read;
int sequenceStart;
int index = 0;
*numberOfSequences = 0;
struct stat64 statStruct;
// check the file size in order to malloc the necessary amount of memory to the fileBuffer array.
if (stat64(inputFile, &statStruct) == -1)
{
return NULL;
}
fileSize = statStruct.st_size;
FILE* file = fopen64(inputFile, "r");
fileBuffer = malloc(fileSize);
while ((c = fgetc(file)) != '\n')
{
if (c == '>')
{
*numberOfSequences += 1;
currentSequence->sequenceName = &fileBuffer[index];
} else if (c == ' ' || c == '\t')
{
fileBuffer[index++] = '\0';
currentSequence->sequenceDescription = &fileBuffer[index];
} else
{
fileBuffer[index++] = c;
}
}
fileBuffer[index++] = '\0';
currentSequence->sequenceType = 1;
currentSequence->data = &fileBuffer[index];
sequenceStart = index;
// Uses fgetc to read the file in one byte at a time.
// Any ASCII character of lesser numerical value than 'A' will be checked if it is '>' to signify
// a new sequence start.
// Any other non-alphabet characters will be ignored in order to skip newlines in the sequence data.
while((c = fgetc(file)) != EOF)
{
if (score->lookupTable[c] < 0)
{
if (c == '>')
{
*numberOfSequences += 1;
currentSequence->sequenceLength = index - sequenceStart;
currentSequence->next = malloc(sizeof(Sequence));
currentSequence = currentSequence->next;
currentSequence->sequenceName = &fileBuffer[index];
currentSequence->sequenceType = 1;
c = fgetc (file);
while (c != '\n' && c != '\r')
{
if (c == ' ' || c == '\t')
{
fileBuffer[index++] = '\0';
currentSequence->sequenceDescription = &fileBuffer[index];
while ((c = fgetc(file)) != '\n' && c != '\r')
{
fileBuffer[index++] = (char) c;
}
} else
{
fileBuffer[index++] = (char) c;
c = fgetc (file);
}
}
fileBuffer[index++] = '\0';
currentSequence->data = &fileBuffer[index];
sequenceStart = index;
} else if (score->lookupTable[c] == -19)
{
// ignore characters: space, tab, carriage return, newline
} else
{
printf("Error: character '%c' (%d) in sequence %d is not defined in the scoring matrix file. Exiting.\n", c, (int)c, *numberOfSequences);
fflush(stdout);
exit(1);
}
} else
{
fileBuffer[index++] = score->lookupTable[c];
}
}
currentSequence->sequenceLength = index - sequenceStart;
return fileBuffer;
}
/*! \brief replace fopen, report error when the file open fails */
inline std::FILE *FopenCheck(const char *fname, const char *flag) {
std::FILE *fp = fopen64(fname, flag);
Check(fp != NULL, "can not open file \"%s\"\n", fname);
return fp;
}
int main(int argc, char *argv[]) {
gotsignal = 0;
signal(SIGINT, breakhandler);
signal(SIGPIPE, breakhandler);
TString inputFile1, inputFile2, outputFile;
Bool_t haveFile1=0, haveFile2=0, haveOutput=0;
Int_t file1zipped = -1; Int_t file2zipped = -1;
Int_t file1Events=0, file2Events=0;
Int_t dataWritten1=0, dataWritten2=0;
TFile *mOut = new TFile("merge.root", "RECREATE");
TH1F *tac = new TH1F("tac", "tac", 1000, -500, 500);
if (argc < 7) {
printf("\nMinimum usage: %s -f1 <Filename1> -f2 <Filename2> -fOut <OutputFilename>\n", argv[0]);
exit(1);
} else if ((strcmp(argv[1], "-h") == 0)) {
printf("\nMinimum usage: %s -f1 <Filename1> -f2 <Filename2> -fOut <OutputFilename>\n", argv[0]);
exit(1);
} else {
Int_t i=1;
while( i<argc ) {
if (strcmp(argv[i], "-f1") == 0) {
haveFile1 = 1; i++;
inputFile1 = argv[i]; i++;
file1zipped = 0;
} else if (strcmp(argv[i], "-f1zip") == 0) {
haveFile1 = 1; i++;
inputFile1 = argv[i]; i++;
file1zipped = 1;
} else if (strcmp(argv[i], "-f2") == 0) {
haveFile2 = 1; i++;
inputFile2 = argv[i]; i++;
file2zipped = 0;
} else if (strcmp(argv[i], "-f2zip") == 0) {
haveFile2 = 1; i++;
inputFile2 = argv[i]; i++;
file2zipped = 1;
} else if (strcmp(argv[i], "-fOut") == 0) {
haveOutput = 1; i++;
outputFile = argv[i]; i++;
} else if (strcmp(argv[i], "-f1ExcludeCrysID") == 0) {
i++;
nFile1Exclude = atoi(argv[i]); i++;
for (Int_t n=0; n<nFile1Exclude; n++) {
file1Exclude[n] = atoi(argv[i]); i++;
}
} else if (strcmp(argv[i], "-h") == 0) {
printf("\nMinimum usage: %s -f1 <Filename1> -f2 <Filename2> -fOut <OutputFilename>\n", argv[0]);
printf(" Option flags: -f1ExcludeCrysID <N> <Value1> ... <ValueN>\n");
printf(" This excludes certain mode2 crystal_id values in File1 from\n");
printf(" being included in the merged output.\n");
printf("\n");
exit(1);
}
}
}
TStopwatch timer;
timer.Start();
if ( !haveFile1 || !haveFile2 || !haveOutput ) {
cout << "Problem: we don't have all the file information required." << endl;
printf("Minimum usage: %s -f1 <Filename1> -f2 <Filename2> -fOut <OutputFilename>", argv[0]);
exit(1);
}
/***********************************************************/
/* MERGED OUTPUT FILE NAME */
/***********************************************************/
int pipeFlag = 0;
/* Set-up merged output file */
FILE *mergeOutFile;
if (!strcmp(outputFile.Data(), "pipe")) {
mergeOutFile = stdout;
pipeFlag = 1;
} else {
mergeOutFile = fopen64(outputFile.Data(), "w");
}
if (!mergeOutFile) {
printf("Cannot open \"%s\" for reading\n", outputFile.Data());
exit(3);
} else {
if (!pipeFlag)
printf("Output file \"%s\" opened\n", outputFile.Data());
}
/***********************************************************/
/* INPUT FILE #1 FOR MERGING */
/***********************************************************/
if (file1zipped) {
inputFile1 = "./GEB_HFC -z -p " + inputFile1;
} else {
inputFile1 = "./GEB_HFC -p " + inputFile1;
}
inFile1 = popen(inputFile1.Data(), "r");
if (!inFile1) {
printf("Cannot open %s for reading\n", inputFile1.Data());
exit(2);
} else {
if (!pipeFlag)
printf("File #1 -- \"%s\" opened for reading\n", inputFile1.Data());
}
/***********************************************************/
/* INPUT FILE #2 FOR MERGING */
/***********************************************************/
if (file2zipped) {
inputFile2 = "./GEB_HFC -z -p " + inputFile2;
} else {
inputFile2 = "./GEB_HFC -p " + inputFile2;
}
inFile2 = popen(inputFile2.Data(), "r");
if (!inFile2) {
printf("Cannot open %s for reading\n", inputFile2.Data());
exit(2);
} else {
if (!pipeFlag)
printf("File #2 -- \"%s\" opened for reading\n", inputFile2.Data());
}
/***********************************************************/
/* Loop and merge files... */
/***********************************************************/
long long int lastTS = 0; int lastTSfile = 0;
while ( !doneFile2 && ! gotsignal) {
GetNextEvent2(dataWritten2);
if ( !doneFile1 ) {
while ( GetNextEvent1(dataWritten1) && !doneFile1 && !gotsignal) {
if ((fileBuf1.header.timestamp) <
(fileBuf2.header.timestamp)) {
//cout << fileBuf1.header.timestamp << " " << fileBuf2.header.timestamp << " -- 1 lower " << endl;
if (fileBuf1.header.timestamp - lastTS < 1000) {
if (lastTSfile == 2) {tac->Fill(fileBuf1.header.timestamp - lastTS);}
} else {
lastTS = fileBuf1.header.timestamp; lastTSfile = 1;
}
/* Write the file 1 event to disk, it's go the lower TS. */
fwrite(&fileBuf1,
fileBuf1.header.length + sizeof(struct globalHeader),
1, mergeOutFile);
file1Events++;
dataWritten1 = 1;
dataWritten2 = 0;
} else {
//cout << fileBuf1.header.timestamp << " " << fileBuf2.header.timestamp << " -- 2 lower " << endl;
//cin.get();
/* Write the file 2 event to disk, it's go the lower TS. */
fwrite(&fileBuf2,
fileBuf2.header.length + sizeof(struct globalHeader),
1, mergeOutFile);
file2Events++;
dataWritten1 = 0;
dataWritten2 = 1;
break;
}
}
} else { /* if (doneFile1) */
/* File 1 is done, just empty out file 2 now */
fwrite(&fileBuf2, fileBuf2.header.length + sizeof(struct globalHeader),
1, mergeOutFile);
file2Events++;
dataWritten2 = 1;
}
}
fclose(inFile2);
if (!doneFile1) {
/* Out of file2 events, dump the remaining file1 events, if any */
while ( GetNextEvent1(dataWritten1) && !doneFile1) {
fwrite(&fileBuf1, fileBuf1.header.length + sizeof(struct globalHeader),
1, mergeOutFile);
file1Events++;
dataWritten1 = 1;
}
}
if (!pipeFlag)
cout << "File 1 events " << file1Events << " "
<< "File 2 events " << file2Events << endl;
fclose(mergeOutFile);
tac->Write();
mOut->Write();
mOut->Close();
return 0;
}
void read_rc( FILE *fp )
{
char *buffer = 0;
char rcfile[256];
char MBM_line[256];
int n;
if ( fp == 0 )
{
LYAddPathToHome( rcfile, 256, ".lynxrc" );
fp = fopen64( rcfile, "r" );
if ( fp )
{
if ( WWW_TraceFlag )
{
fprintf( TraceFP( ), "read_rc opened %s\n", rcfile );
while ( LYSafeGets( &buffer, fp ) == 0 )
{
char *name, *value, *notes;
Config_Type *tbl;
ParseUnion *q;
LYTrimTrailing( buffer );
name = LYSkipBlanks( buffer );
if ( !( *(short*)(*(int*)(__ctype_b_loc( )) + ( name[0] * 2 )) & 4 ) && name[0] )
{
value = strchr( name, '=' );
if ( value == 0 )
{
if ( WWW_TraceFlag )
{
fprintf( TraceFP( ), "LYrcFile: missing '=' %s\n", name );
}
}
else
{
value[0] = 0;
value++;
LYTrimTrailing( name );
value = LYSkipBlanks( value );
if ( WWW_TraceFlag && ( WWW_TraceMask & 8 ) )
{
fprintf( TraceFP( ), "LYrcFile %s:%s\n", name, value );
}
tbl = lookup_config( name );
if ( tbl->name[0] == 0 )
{
char *special = "multi_bookmark";
if ( strncasecomp( name, special, strlen( special ) ) == 0 )
{
tbl = lookup_config( special );
}
if ( tbl->name[0] == 0 )
{
if ( WWW_TraceFlag )
{
fprintf( TraceFP( ), "LYrcFile: ignored %s=%s\n", name, value );
}
}
}
q = &tbl->value;
switch ( tbl->type )
{
case 0:
break;
case 2:
if ( q->add_value[0] )
{
q->override_primary_action = getBool( value );
}
break;
case 3:
if ( q->add_value[0] )
{
;
}
break;
case 1:
n = 0;
for ( ; tbl->strings[ n ]; n++ )
{
if ( strcasecomp( value, &tbl->strings[ n ] ) == 0 )
{
q->add_value = &n;
}
else
{
// n++;
}
}
break;
case 5:
if ( tbl->table )
{
LYgetEnum( &tbl->table, value, (int*)q->add_value[0] );
}
break;
case 4:
if ( q->add_value[0] && sscanf( value, value, "%d", ival ) == 1 )
{
q->add_value = &ival;
}
break;
case 6:
if ( q->add_value[0] )
{
if ( q->add_value )
HTSACat( (int)( &q->add_value[0]->next ), "," );
HTSACat( (int)( &q->add_value[0]->next ), value );
}
break;
case 7:
n = 1;
for ( ; n <= 25; n++ )
{
sprintf( MBM_line, "multi_bookmark%c", LYindex2MBM( n ) );
if ( strcasecomp( name, MBM_line ) == 0 )
{
notes = strchr( value, ',' );
if ( notes )
{
notes = 0;
notes++;
LYTrimTrailing( value );
notes = LYSkipBlanks( notes );
}
else
{
notes = value + strlen( value );
}
HTSACopy( &MBM_A_subbookmark[ n ], value );
HTSACopy( &MBM_A_subdescript[ n ], notes );
}
else
{
// n++;
}
}
break;
case 8:
if ( q->add_value[0] )
{
HTSACopy( (int)( &q->add_value[0]->next ), value );
}
break;
}
}
}
}
}
}
}
else
if ( WWW_TraceFlag )
{
fprintf( TraceFP( ), "read_rc used passed-in stream\n" );
}
return;
}
size_t read_alias_file( unsigned char *fname, int fname_len )
{
int ecx;
FILE *fp;
size_t added;
*(int*)(mempcpy( mem_alloc( fname_len + 14 ), &fname[0], fname_len )) = 0x636f6c2f;
*(int*)(mempcpy( mem_alloc( fname_len + 14 ), &fname[0], fname_len ) + 4) = 0x2e656c61;
*(int*)(mempcpy( mem_alloc( fname_len + 14 ), &fname[0], fname_len ) + 8) = 0x61696c61;
*(short*)(mempcpy( mem_alloc( fname_len + 14 ), &fname[0], fname_len ) + 12) = 115;
fp = fopen64( *ebp_16472, "rb" );
mem_free( *ebp_16472 );
added = 0;
if ( fp[0]._flags )
{
while ( !( ( fp->_flags/*.1_1of4*/ & 16 ) & 255 ) && fgets_unlocked( buf, 8192, &fp[0] ) )
{
if ( strchr( buf, 10 ) == 0 )
{
do
{
}
while ( !fgets_unlocked( ebp_16412, 8192, &fp[0] ) || strchr( ebp_16412, 10 ) );
}
{
unsigned char altbuf[8192];
cp[0] = buf[0];
if ( cp[0] == '#' || ( cp[0] & 255 ) == 0 )
continue;
else
{
cp[0] = cp[0] + 1;
if ( cp[1] )
{
cp[0] = 0;
cp[0] = cp[1];
}
else
cp[0] = cp[0];
if ( ( cp[0] & 255 ) == 0 )
continue;
else
{
size_t alias_len;
size_t value_len;
if ( cp[1] )
{
if ( cp[1] == 10 )
{
cp[1] = 0;
cp[1] = 10;
}
else
cp[1] = 0;
}
if ( maxmap <= nmap )
{
if ( realloc( map, 800 ? 800 << 4 : 800 ) )
{
map = (struct sgml_node_info*)realloc( map, 800 ? 800 << 4 : 800 );
maxmap = edx;
}
}
alias_len = strlen( &cp[0] ) + 1;
value_len = strlen( &cp[0] ) + 1;
if ( string_space_act + value_len + alias_len <= string_space_max )
{
}
else
{
size_t new_size = new_pool[ (unsigned char)( ( value_len + alias_len < 1024 ) ^ 1 ) ? 1024 : value_len + alias_len ];
unsigned char *new_pool = (unsigned char*)realloc( string_space, new_pool[ (unsigned char)( ( value_len + alias_len < 1024 ) ^ 1 ) ? 1024 : value_len + alias_len ] );
if ( new_pool )
{
if ( string_space == new_pool )
{
}
else
{
size_t i;
if ( nmap )
{
string_space[0] = new_pool - string_space;
i = 0;
do
{
map->string.length += new_pool - string_space;
map->string.string += new_pool - string_space;
i++;
}
while ( i + 1 < nmap );
}
else
{
}
}
string_space_max = new_size;
string_space = new_pool;
}
}
map->string.length = memcpy( string_space + added, &cp[0] );
string_space_act += ebp_16444;
map->string.string = (unsigned char*)memcpy( &string_space[ ebp_16444 + string_space_act ], &cp[0] );
string_space_act += ebp_16440;
nmap++;
added++;
}
}
}
}
fclose( &fp[0] );
if ( added )
qsort( map, nmap, 8, &alias_compare );
}
if ( 0 ^ 0 )
{
__stack_chk_fail( );
}
return added;
}
unsigned int ProcessSearchPattern() {
FILE *searchPatternFile;
char c;
unsigned int i, j, n;
searchPatternFile = fopen64(PatternFileName, "r");
if (searchPatternFile == NULL) {
fprintf(stderr, "Cannot open search pattern file : %s\n", PatternFileName);
exit(1);
}
j = 0;
n = 0;
// Find out the size of search pattern
while (!feof(searchPatternFile)) {
c = (char)fgetc(searchPatternFile);
// Find the first comment line
while (!feof(searchPatternFile) && c != '>') {
c = (char)fgetc(searchPatternFile);
}
while (!feof(searchPatternFile)) {
// Skip comment line
while (!feof(searchPatternFile) && c != '\n') {
c = (char)fgetc(searchPatternFile);
}
// Count no. of characters
while (!feof(searchPatternFile) && c != '>') {
if (c != '\n') {
j++;
}
c = (char)fgetc(searchPatternFile);
}
// Count no. of patterns
n++;
}
}
if (j > searchPatternAllocated) {
if (searchPattern != NULL) {
MMUnitFree(searchPattern, searchPatternAllocated);
}
searchPattern = MMUnitAllocate(j);
searchPatternAllocated = j;
}
if (n + 1 > searchPatternPositionAllocated) {
if (searchPatternPosition != NULL) {
MMUnitFree(searchPatternPosition, searchPatternPositionAllocated);
}
searchPatternPosition = MMUnitAllocate((n + 1) * sizeof(unsigned int));
searchPatternPositionAllocated = n + 1;
}
fseek(searchPatternFile, 0, SEEK_SET);
j = 0;
n = 0;
i = 0;
searchPatternPosition[0] = 0;
// Read pattern into memory
while (!feof(searchPatternFile)) {
c = (char)fgetc(searchPatternFile);
// Find the first comment line
while (!feof(searchPatternFile) && c != '>') {
c = (char)fgetc(searchPatternFile);
}
while (!feof(searchPatternFile)) {
// Skip comment line
while (!feof(searchPatternFile) && c != '\n') {
c = (char)fgetc(searchPatternFile);
}
// Store pattern
while (!feof(searchPatternFile) && c != '>') {
if (c != '\n') {
searchPattern[j] = c;
j++;
}
c = (char)fgetc(searchPatternFile);
}
// Store pointers to pattern
n++;
searchPatternPosition[n] = j;
}
}
fclose(searchPatternFile);
return n;
}
int minizip_zip(const char * lpszzipfilename,
const char * lpszfilename,
const char * lpszpassword,
int nmode, int nLevel)
{
int nret = -1;
int nstatus = 0;
// int opt_compress_level = Z_DEFAULT_COMPRESSION;
if (nLevel != Z_DEFAULT_COMPRESSION && nLevel != Z_BEST_COMPRESSION && nLevel != Z_BEST_SPEED && nLevel != Z_NO_COMPRESSION)
{
return -10086;
}
int nzipmode = APPEND_STATUS_CREATE;
if (nmode ==0)
{
nzipmode = APPEND_STATUS_CREATE;
}
else
{
nzipmode = APPEND_STATUS_ADDINZIP;
}
// Create archive zipfilename
zipFile zf = zipOpen64(lpszzipfilename, nzipmode);
if (zf == NULL)
{
nstatus = ERROR_CREATE_ZIP;
}
int size_buf = WRITE_BUFFER_SIZE;
Bytef* buf = (Bytef*) malloc(size_buf);
// Get information about the file on disk so we can store it in zip
zip_fileinfo zi;
memset(&zi,0x00,sizeof(zip_fileinfo));
getFileTime(lpszfilename, &zi.tmz_date, &zi.dosDate);
unsigned long crcFile = 0;
if (nstatus == ZIP_OK)
{
nstatus = getCRC32(lpszfilename, buf, size_buf, &crcFile);
}
int zip64 = isLargeFile(lpszfilename);
// Construct the filename that our file will be stored in the zip as.
const char *savefilenameinzip = lpszfilename;
{
const char *tmpptr = NULL;
const char *lastslash = 0;
for (tmpptr = savefilenameinzip; *tmpptr; tmpptr++)
{
if (*tmpptr == '\\' || *tmpptr == '/')
{
lastslash = tmpptr;
}
}
if (lastslash != NULL)
{
savefilenameinzip = lastslash + 1;
}
}
if ((lpszpassword) && (lpszpassword[0] ==0))
{
lpszpassword = NULL;
}
// Create zip file
nstatus = zipOpenNewFileInZip3_64(zf, savefilenameinzip, &zi, NULL, 0, NULL, 0, NULL /* comment*/,
Z_DEFLATED, nLevel, 0,
-MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, lpszpassword, crcFile, zip64);
// Add file to zip
FILE *fin = NULL;
if (nstatus == ZIP_OK)
{
fin = fopen64(lpszfilename, "rb");
if (fin == NULL)
{
nstatus = ERROR_FILE_NOT_FOUND;
}
}
int size_read = 0;
if (nstatus == ZIP_OK)
{
// Read contents of file and write it to zip
do
{
size_read = (int) fread(buf, 1, size_buf, fin);
if ((size_read < size_buf) && (feof(fin) == 0))
{
nstatus = ERROR_WHILE_READ;
}
if (size_read > 0)
{
nstatus = zipWriteInFileInZip(zf, buf, size_read);
}
}
while ((nstatus == ZIP_OK) && (size_read > 0));
}
if (fin)
{
fclose(fin);
}
if (nstatus >= 0)
{
nstatus = zipCloseFileInZip(zf);
}
zipClose(zf, NULL);
// Release memory
free(buf);
if (nstatus == UNZ_OK)
{
nret = 0;
}
return nret;
}
inline FILE* safeFopen(const std::string& path) {
FILE* res = fopen64(path.c_str(), "r");
if(!res)
CERR << "failed to open a file at:" << path << EFATAL;
return res;
}
int do_extract_currentfile(unzFile uf, TCHAR *dirToExtractTo)
{
TCHAR filename_inzip[256];
TCHAR* filename_withoutpath;
TCHAR* p;
int err = UNZ_OK;
FILE *fout = NULL;
void *buf;
uInt size_buf;
TCHAR write_filename[MAX_PATH];
unz_file_info64 file_info;
uLong ratio = 0;
err = unzGetCurrentFileInfo64(uf, &file_info, filename_inzip, sizeof(filename_inzip), NULL, 0, NULL, 0);
if (err != UNZ_OK)
{
_tprintf(_T("error %d with zipfile in unzGetCurrentFileInfo\n"), err);
return err;
}
size_buf = WRITEBUFFERSIZE;
buf = (void *) malloc(size_buf);
if (buf == NULL)
{
_tprintf(_T("Error allocating memory\n"));
return UNZ_INTERNALERROR;
}
p = filename_withoutpath = filename_inzip;
while ((*p) != '\0')
{
if (((*p)=='/') || ((*p)=='\\'))
filename_withoutpath = p+1;
p++;
}
//make sure this exists
_tmkdir(dirToExtractTo);
_tcscpy(write_filename, dirToExtractTo);
_tcscat(write_filename, _T("/"));
_tcscat(write_filename, filename_withoutpath);
err = unzOpenCurrentFile(uf);
if (err != UNZ_OK)
{
_tprintf(_T("error %d with zipfile in unzOpenCurrentFile\n"), err);
}
if (err == UNZ_OK)
{
fout = fopen64(write_filename, _T("wb"));
if (fout == NULL)
{
_tprintf(_T("error opening %s\n"), write_filename);
}
}
if (fout != NULL)
{
_tprintf(_T(" extracting: %s\n"), write_filename);
do
{
err = unzReadCurrentFile(uf, buf, size_buf);
if (err <0)
{
_tprintf(_T("error %d with zipfile in unzReadCurrentFile\n"), err);
break;
}
if (err > 0)
if (fwrite(buf, err, 1, fout) != 1)
{
_tprintf(_T("error in writing extracted file\n"));
err = UNZ_ERRNO;
break;
}
} while (err > 0);
if (fout)
fclose(fout);
/*if (err == 0)
change_file_date(write_filename,file_info.dosDate, file_info.tmu_date);*/
}
if (err == UNZ_OK)
{
err = unzCloseCurrentFile (uf);
if (err != UNZ_OK)
{
_tprintf(_T("error %d with zipfile in unzCloseCurrentFile\n"), err);
}
}
else
unzCloseCurrentFile(uf); /* don't lose the error */
free(buf);
return err;
}
int http_get_timeout (const char * url, const char * outfile, int total_retries, int first_timeout, int connect_timeout, int total_timeout)
{
int code = ERROR;
logprintfl (EUCAINFO, "http_get(): downloading %s\n", outfile);
logprintfl (EUCAINFO, " from %s\n", url);
/* isolate the PATH in the URL as it will be needed for signing */
if (strncasecmp (url, "http://", 7)!=0) {
logprintfl (EUCAERROR, "http_get(): URL must start with http://...\n");
return code;
}
FILE * fp = fopen64 (outfile, "w");
if (fp==NULL) {
logprintfl (EUCAERROR, "http_get(): failed to open %s for writing\n", outfile);
return code;
}
CURL * curl;
CURLcode result;
curl = curl_easy_init ();
if (curl==NULL) {
logprintfl (EUCAERROR, "http_get(): could not initialize libcurl\n");
fclose(fp);
return code;
}
char error_msg [CURL_ERROR_SIZE];
curl_easy_setopt (curl, CURLOPT_ERRORBUFFER, error_msg);
curl_easy_setopt (curl, CURLOPT_URL, url);
// curl_easy_setopt (curl, CURLOPT_HEADERFUNCTION, write_header);
curl_easy_setopt (curl, CURLOPT_HTTPGET, 1L);
/* set up the default write function, but possibly override it below, if compression is desired and possible */
struct write_request params;
params.fp = fp;
curl_easy_setopt (curl, CURLOPT_WRITEDATA, ¶ms);
curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_data);
if (connect_timeout > 0) {
curl_easy_setopt (curl, CURLOPT_CONNECTTIMEOUT, connect_timeout);
}
if (total_timeout > 0) {
curl_easy_setopt (curl, CURLOPT_TIMEOUT, total_timeout);
}
// curl_easy_setopt (curl, CURLOPT_HTTPHEADER, headers); /* register headers */
logprintfl (EUCADEBUG, "http_get(): writing %s output to %s\n", "GET", outfile);
int retries = total_retries;
int timeout = first_timeout;
do {
params.total_wrote = 0L;
params.total_calls = 0L;
result = curl_easy_perform (curl); /* do it */
logprintfl (EUCADEBUG, "http_get(): wrote %ld bytes in %ld writes\n", params.total_wrote, params.total_calls);
if (result) { // curl error (connection or transfer failed)
logprintfl (EUCAERROR, "http_get(): %s (%d)\n", error_msg, result);
} else {
long httpcode;
curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &httpcode);
/* TODO: pull out response message, too */
switch (httpcode) {
case 200L: /* all good */
logprintfl (EUCAINFO, "http_get(): saved image in %s\n", outfile);
code = OK;
break;
case 408L: /* timeout, retry */
logprintfl (EUCAWARN, "http_get(): server responded with HTTP code %ld (timeout)\n", httpcode);
//logcat (EUCADEBUG, outfile); /* dump the error from outfile into the log */
break;
case 404L:
logprintfl (EUCAWARN, "http_get(): server responded with HTTP code %ld (file not found)\n", httpcode);
break;
default: /* some kind of error */
logprintfl (EUCAERROR, "http_get(): server responded with HTTP code %ld\n", httpcode);
//logcat (EUCADEBUG, outfile); /* dump the error from outfile into the log */
retries=0;
}
}
if (code!=OK && retries>0) {
logprintfl (EUCAERROR, " download retry %d of %d will commence in %d seconds\n", retries, total_retries, timeout);
sleep (timeout);
fseek (fp, 0L, SEEK_SET);
timeout <<= 1;
}
retries--;
} while (code!=OK && retries>0);
fclose (fp);
if ( code != OK ) {
logprintfl (EUCAINFO, "http_get(): due to error, removing %s\n", outfile);
remove (outfile);
}
// curl_slist_free_all (headers);
curl_easy_cleanup (curl);
return code;
}
void handle_client(int fd, const char * userTest, const char * passTest) {
FILE * fp = fdopen(fd, "w+");
char * user, * pass, * path;
long long initial;
if (read_client_request(fp, &user, &pass, &path, &initial) < 0) {
return;
}
if (strcmp(user, userTest) != 0 || strcmp(pass, passTest) != 0) {
respond_error(fp, PROTOCOL_ERROR_AUTH, "Login incorrect.");
goto handleError;
}
#ifdef __linux__
struct stat64 info;
if (stat64(path, &info) < 0) {
#else
struct stat info
if (stat(path, &info) < 0) {
#endif
respond_error(fp, PROTOCOL_ERROR_NO_FILE, "Could not stat file.");
goto handleError;
}
if (S_ISDIR(info.st_mode)) {
respond_error(fp, PROTOCOL_ERROR_NOT_FILE, "Could not send directory.");
goto handleError;
}
if (initial > info.st_size) {
respond_error(fp, PROTOCOL_ERROR_INVALID_INITIAL, "Invalid initial offset.");
goto handleError;
}
#ifdef __linux__
FILE * readFile = fopen64(path, "r");
#else
FILE * readfile = fopen(path, "r");
#endif
if (!readFile) {
perror("fopen");
respond_error(fp, PROTOCOL_ERROR_INTERNAL, "fopen() failed");
goto handleError;
}
if (respond_success(fp, info.st_size - initial) < 0) {
fclose(readFile);
goto handleError;
}
// read file from start offset
fseek(readFile, initial, SEEK_SET);
char buff[512];
while (!feof(readFile)) {
int got = fread(buff, 1, 512, readFile);
if (got == 0) break;
if (fwrite(buff, 1, got, fp) != got) {
break;
}
}
fclose(readFile);
handleError:
fclose(fp);
free(user);
free(pass);
free(path);
return;
}
void sigchild_handler(int signal) {
int status;
wait(&status);
}
int InCoreInterp::outputFile(char *outputName, int outputFormat, unsigned int outputType)
{
int i,j,k;
FILE **arcFiles;
char arcFileName[1024];
FILE **gridFiles;
char gridFileName[1024];
char *ext[5] = {".min", ".max", ".mean", ".idw", ".den"};
unsigned int type[5] = {OUTPUT_TYPE_MIN, OUTPUT_TYPE_MAX, OUTPUT_TYPE_MEAN, OUTPUT_TYPE_IDW, OUTPUT_TYPE_DEN};
int numTypes = 5;
// open ArcGIS files
if(outputFormat == OUTPUT_FORMAT_ARC_ASCII || outputFormat == OUTPUT_FORMAT_ALL)
{
if((arcFiles = (FILE **)malloc(sizeof(FILE *) * numTypes)) == NULL)
{
cout << "Arc File open error: " << endl;
return -1;
}
for(i = 0; i < numTypes; i++)
{
if(outputType & type[i])
{
strncpy(arcFileName, outputName, sizeof(arcFileName));
strncat(arcFileName, ext[i], strlen(ext[i]));
strncat(arcFileName, ".asc", strlen(".asc"));
if((arcFiles[i] = fopen64(arcFileName, "w+")) == NULL)
{
cout << "File open error: " << arcFileName << endl;
return -1;
}
}else{
arcFiles[i] = NULL;
}
}
} else {
arcFiles = NULL;
}
// open Grid ASCII files
if(outputFormat == OUTPUT_FORMAT_GRID_ASCII || outputFormat == OUTPUT_FORMAT_ALL)
{
if((gridFiles = (FILE **)malloc(sizeof(FILE *) * numTypes)) == NULL)
{
cout << "File array allocation error" << endl;
return -1;
}
for(i = 0; i < numTypes; i++)
{
if(outputType & type[i])
{
strncpy(gridFileName, outputName, sizeof(arcFileName));
strncat(gridFileName, ext[i], strlen(ext[i]));
strncat(gridFileName, ".grid", strlen(".grid"));
if((gridFiles[i] = fopen64(gridFileName, "w+")) == NULL)
{
cout << "File open error: " << gridFileName << endl;
return -1;
}
}else{
gridFiles[i] = NULL;
}
}
} else {
gridFiles = NULL;
}
//struct tms tbuf;
//clock_t t0, t1;
//t0 = times(&tbuf);
// print ArcGIS headers
if(arcFiles != NULL)
{
for(i = 0; i < numTypes; i++)
{
if(arcFiles[i] != NULL)
{
fprintf(arcFiles[i], "ncols %d\n", GRID_SIZE_X);
fprintf(arcFiles[i], "nrows %d\n", GRID_SIZE_Y);
fprintf(arcFiles[i], "xllcorner %f\n", min_x);
fprintf(arcFiles[i], "yllcorner %f\n", min_y);
fprintf(arcFiles[i], "cellsize %f\n", GRID_DIST_X);
fprintf(arcFiles[i], "NODATA_value -9999\n");
}
}
}
// print Grid headers
if(gridFiles != NULL)
{
for(i = 0; i < numTypes; i++)
{
if(gridFiles[i] != NULL)
{
fprintf(gridFiles[i], "north: %f\n", max_y);
fprintf(gridFiles[i], "south: %f\n", min_y);
fprintf(gridFiles[i], "east: %f\n", max_x);
fprintf(gridFiles[i], "west: %f\n", min_x);
fprintf(gridFiles[i], "rows: %d\n", GRID_SIZE_Y);
fprintf(gridFiles[i], "cols: %d\n", GRID_SIZE_X);
}
}
}
// print data
for(i = GRID_SIZE_Y - 1; i >= 0; i--)
{
for(j = 0; j < GRID_SIZE_X; j++)
{
if(arcFiles != NULL)
{
// Zmin
if(arcFiles[0] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(arcFiles[0], "-9999 ");
else
fprintf(arcFiles[0], "%f ", interp[j][i].Zmin);
}
// Zmax
if(arcFiles[1] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(arcFiles[1], "-9999 ");
else
fprintf(arcFiles[1], "%f ", interp[j][i].Zmax);
}
// Zmean
if(arcFiles[2] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(arcFiles[2], "-9999 ");
else
fprintf(arcFiles[2], "%f ", interp[j][i].Zmean);
}
// Zidw
if(arcFiles[3] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(arcFiles[3], "-9999 ");
else
fprintf(arcFiles[3], "%f ", interp[j][i].Zidw);
}
// count
if(arcFiles[4] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(arcFiles[4], "-9999 ");
else
fprintf(arcFiles[4], "%d ", interp[j][i].count);
}
}
if(gridFiles != NULL)
{
// Zmin
if(gridFiles[0] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(gridFiles[0], "-9999 ");
else
fprintf(gridFiles[0], "%f ", interp[j][i].Zmin);
}
// Zmax
if(gridFiles[1] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(gridFiles[1], "-9999 ");
else
fprintf(gridFiles[1], "%f ", interp[j][i].Zmax);
}
// Zmean
if(gridFiles[2] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(gridFiles[2], "-9999 ");
else
fprintf(gridFiles[2], "%f ", interp[j][i].Zmean);
}
// Zidw
if(gridFiles[3] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(gridFiles[3], "-9999 ");
else
fprintf(gridFiles[3], "%f ", interp[j][i].Zidw);
}
// count
if(gridFiles[4] != NULL)
{
if(interp[j][i].empty == 0 &&
interp[j][i].filled == 0)
fprintf(gridFiles[4], "-9999 ");
else
fprintf(gridFiles[4], "%d ", interp[j][i].count);
}
}
}
if(arcFiles != NULL)
for(k = 0; k < numTypes; k++)
{
if(arcFiles[k] != NULL)
fprintf(arcFiles[k], "\n");
}
if(gridFiles != NULL)
for(k = 0; k < numTypes; k++)
{
if(gridFiles[k] != NULL)
fprintf(gridFiles[k], "\n");
}
}
// close files
if(gridFiles != NULL)
{
for(i = 0; i < numTypes; i++)
{
if(gridFiles[i] != NULL)
fclose(gridFiles[i]);
}
}
if(arcFiles != NULL)
{
for(i = 0; i < numTypes; i++)
{
if(arcFiles[i] != NULL)
fclose(arcFiles[i]);
}
}
//t1 = times(&tbuf);
//printf("elapsed time: %10.10f\n", (t1 - t0)/(double) sysconf(_SC_CLK_TCK));
return 0;
}
BOOLEAN class_cp_is_entry_resolved(Class_Handle clazz, U_16 cp_index) {
ConstantPool& cp = clazz->get_constant_pool();
#ifdef ORDER
bool res = cp.is_entry_resolved(cp_index, (Class *)clazz);
#else
bool res = cp.is_entry_resolved(cp_index);
#endif
if (!res) {
unsigned char tag = cp.get_tag(cp_index);
//during the loading of a class not all items in it's constant pool are updated
if (tag == CONSTANT_Fieldref || tag == CONSTANT_Methodref
|| tag == CONSTANT_InterfaceMethodref || tag == CONSTANT_Class)
{
uint16 typeIndex = tag == CONSTANT_Class ? cp_index : cp.get_ref_class_index(cp_index);
#ifdef ORDER
res = cp.is_entry_resolved(typeIndex, (Class *)clazz);
#else
res = cp.is_entry_resolved(typeIndex);
#endif
if (!res) {
// the type is not marked as loaded in local constant pool
// ask classloader directly
uint16 nameIdx = cp.get_class_name_index(typeIndex);
String* typeName = cp.get_utf8_string(nameIdx);
assert(typeName!=NULL);
#ifdef ORDER
Class* type = NULL;
if(vm_order_record){ // record mode
type = clazz->get_class_loader()->LookupClass(typeName);
U_32 tid = hythread_self()->thread_id;
if (order_system_call[tid] == NULL)
{
char name[40];
sprintf(name, "SYSTEM_CALL.%d.log", tid);
order_system_call[tid] = fopen64(name, "a+");
#ifdef ORDER_DEBUG
assert(order_system_call[tid]);
#endif
}
#ifdef ORDER_DEBUG
fprintf(order_system_call[tid], "[%d] ", 38);
#endif
fprintf(order_system_call[tid], "%d\n", type == NULL ? 0 : 1);
}
else{ // replay mode
U_32 tid = hythread_self()->thread_id;
if (order_system_call[tid] == NULL)
{
char name[40];
sprintf(name, "SYSTEM_CALL.%d.log", tid);
order_system_call[tid] = fopen64(name, "r");
#ifdef ORDER_DEBUG
assert(order_system_call[tid]);
#endif
}
#ifdef ORDER_DEBUG
int bit_num;
fscanf(order_system_call[tid], "[%d] ", &bit_num);
assert(bit_num == 38);
#endif
int type_state = 0;
fscanf(order_system_call[tid], "%d\n", &type_state);
if(type_state == 0){
type = NULL;
}
else{
while((type = clazz->get_class_loader()->LookupClass(typeName)) == NULL){
#ifdef ORDER_DEBUG
printf("(type = clazz->get_class_loader()->LookupClass(typeName)) == NULL!!! \n");
#endif
usleep(100);
}
#ifdef ORDER_DEBUG
assert(type);
#endif
}
}
#else //NOT define ORDER
Class* type = clazz->get_class_loader()->LookupClass(typeName);
#endif //#ifdef ORDER
if (type) {
/*TODO: uncommenting this code lead to a crash in StressLoader test
clazz->lock();
cp.resolve_entry(typeIndex, type);
clazz->unlock();*/
res = true;
}
//if array of primitives -> return true;
if (*typeName->bytes=='[' && !strchr(typeName->bytes, 'L')) {
return true;
}
}
}
}
return res;
}