/**
* @brief when unpack end, do not delete all the unpacked files
* the package is xx.tar/xxx.rpm type
* multy process
*/
void testUnunpackEntryNormalMultyProcess()
{
int Pid = fork();
if (Pid < 0)
{
fprintf(stderr,"Fork error!\n");
exit(-1);
}
else if (Pid == 0)
{
int argc = 7;
char *argv[] = {"../ununpack", "-qCR", "-m", "5", "./test-data/testdata4unpack/rpm.tar", "-d", "./test-result/"};
deleteTmpFiles("./test-result/");
existed = file_dir_existed("./test-result/");
CU_ASSERT_EQUAL(existed, 0); // ./test-result/ does not exist
UnunpackEntry(argc, argv);
exit(0);
} else
{
int status;
waitpid(Pid, &status, 0);
existed = file_dir_existed("./test-result/rpm.tar.dir/yast2-trans-bn.rpm.unpacked.dir/yast2-trans-bn.rpm.dir/usr/share/doc/packages/yast2-trans-bn/status.txt");
CU_ASSERT_EQUAL(existed, 1); // the file above exists
}
}
/**
* @brief when unpack end, delete all the unpacked files
*/
void testUnunpackEntryNormalDeleteResult()
{
int Pid = fork();
if (Pid < 0)
{
fprintf(stderr,"Fork error!\n");
exit(-1);
}
else if (Pid == 0)
{
int argc = 5;
char *argv[] = {"../ununpack", "-qCRx", "./test-data/testdata4unpack.7z", "-d", "./test-result/"};
deleteTmpFiles("./test-result/");
existed = file_dir_existed("./test-result/");
CU_ASSERT_EQUAL(existed, 0); // ./test-result/ does not exist
int rc = system("mkdir ./test-result; cp ./test-data/testdata4unpack/ununpack.c.Z ./test-result");
if (rc != 0) exit(-1);
UnunpackEntry(argc, argv);
exit(0);
} else
{
int status;
waitpid(Pid, &status, 0);
existed = file_dir_existed("./test-result/testdata4unpack.7z.dir/testdata4unpack/libfossagent.a.dir/libfossagent.o");
CU_ASSERT_EQUAL(existed, 0); // the file above does not exist
existed = file_dir_existed("./test-result/");
CU_ASSERT_EQUAL(existed, 0); // ./test-result/ does not exist
}
}
/**
* @brief test the partition file
*/
void testTraverseChild4PartitionFile()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
Filename = "../test-data/testdata4unpack/vmlinuz-2.6.26-2-686";
MkDirs("./test-result/vmlinuz-2.6.26-2-686.dir/");
strcpy(Queue[0].ChildRecurse, "./test-result/vmlinuz-2.6.26-2-686.dir/");
lstat(Filename, &Stat);
ContainerInfo CITemp;
memset(&CITemp,0,sizeof(ContainerInfo));
strcpy(CITemp.Source, Filename);
strcpy(CITemp.Partdir, NewDir);
strcpy(CITemp.Partname, "vmlinuz-2.6.26-2-686");
strcpy(CITemp.PartnameNew, "vmlinuz-2.6.26-2-686");
ParentInfo PITemp = {26, 1287725739, 1287725739, 0};
lstat(Filename, &Stat);
CITemp.Stat = Stat;
CITemp.PI = PITemp;
int Pid;
Pid = fork();
if (Pid == 0)
{
TraverseChild(Index, &CITemp, NewDir);
} else
{
ParentWait();
exists = file_dir_exists("./test-result/vmlinuz-2.6.26-2-686.dir/Partition_0000");
FO_ASSERT_EQUAL(exists, 1); // existing
}
}
/**
* @brief the option is invalid
*/
void testUnunpackEntryOptionInvalid()
{
int Pid = fork();
if (Pid < 0)
{
fprintf(stderr,"Fork error!\n");
exit(-1);
}
else if (Pid == 0)
{
int argc = 5;
char *argv[] = {"../ununpack", "-qCRs",
"./test-data/testdata4unpack/rpm.tar", "-d", "./test-result/"}; // option H is invalid
deleteTmpFiles("./test-result/");
existed = file_dir_existed("./test-result/");
CU_ASSERT_EQUAL(existed, 0); // ./test-result/ does not exist
UnunpackEntry(argc, argv);
exit(0);
} else
{
int status;
waitpid(Pid, &status, 0);
int code = WEXITSTATUS(status);
existed = file_dir_existed("./test-result/rpm.tar.dir/yast2-trans-bn.rpm.unpacked.dir/yast2-trans-bn.rpm.dir/usr/share/doc/packages/yast2-trans-bn/status.txt");
CU_ASSERT_EQUAL(existed, 0); // the file above does not exist
CU_ASSERT_EQUAL(code, 25); // the exit code of unpack agent is 25
}
}
/**
* @brief abnormal parameters
*/
void testExtractAR4EmptyParameters()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
FO_ASSERT_EQUAL(exists, 0); // not existing
Result = ExtractAR("", ""); // empty parameters
FO_ASSERT_EQUAL(Result, 1); // fail to Extract archieve library
}
/**
* @brief abnormal parameters
*/
void testExtractAR4ErrorParameters()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
FO_ASSERT_EQUAL(exists, 0); // not existing
MkDirs("./test-result/fossology-1.2.0-1.el5.i386.rpm.dir/");
Filename = "../test-data/testdata4unpack/fossology-1.2.0-1.el5.i386.rpm";
Result = ExtractAR(Filename, "./test-result/fossology-1.2.0-1.el5.i386.rpm.dir");
FO_ASSERT_EQUAL(Result, 1); // fail to Extract archieve library
}
/**
* @brief test traversestart dirctory
*/
void testTraverseStartDir()
{
Filename = "../test-data/testdata4unpack/testdir/";
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/test.jar.dir/ununpack");
FO_ASSERT_EQUAL(exists, 0); // ./test-result/ is not existing
TraverseStart(Filename, Label, NewDir, Recurse);
exists = file_dir_exists("./test-result/test.jar.dir/ununpack");
FO_ASSERT_EQUAL(exists, 1); // ./test-result/ is existing
}
void testTraverseStartNormal()
{
Filename = "../test-data/testdata4unpack/threezip.zip";
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/threezip.zip.dir/twozip.zip.dir/Desktop.zip.dir/record.txt");
FO_ASSERT_EQUAL(exists, 0); // ./test-result/ is not existing
TraverseStart(Filename, Label, NewDir, Recurse);
exists = file_dir_exists("./test-result/threezip.zip.dir/twozip.zip.dir/Desktop.zip.dir/record.txt");
FO_ASSERT_EQUAL(exists, 1); // ./test-result/ is existing
}
/**
* @brief ununpack iso file
*/
void testTraverseChild4IsoFile()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
Filename = "../test-data/testdata4unpack/imagefile.iso";
MkDirs("./test-result/imagefile.iso.dir/");
lstat(Filename, &Stat);
ContainerInfo CITemp;
memset(&CITemp,0,sizeof(ContainerInfo));
strcpy(CITemp.Source, Filename);
strcpy(CITemp.Partdir, NewDir);
strcpy(CITemp.Partname, "imagefile.iso");
strcpy(CITemp.PartnameNew, "imagefile.iso.dir");
CITemp.Stat = Stat;
CITemp.PI.Cmd = 20;
CITemp.PI.StartTime = 1287725739;
CITemp.PI.EndTime = 1287725739;
CITemp.PI.ChildRecurseArtifact = 0;
CITemp.uploadtree_pk = 0;
CITemp.Artifact = 0;
CITemp.IsDir = 0;
CITemp.IsCompressed = 0;
CITemp.uploadtree_pk = 0;
CITemp.pfile_pk = 0;
CITemp.ufile_mode = 0;
strcpy(Queue[0].ChildRecurse, "./test-result/imagefile.iso.dir");
/* test TraverseChild */
int Pid;
Pid = fork();
if (Pid == 0)
{
TraverseChild(Index, &CITemp, NewDir);
} else
{
ParentWait();
int rc = 0;
char commands[250];
sprintf(commands, "isoinfo -f -R -i '%s' | grep ';1' > /dev/null ", Filename);
rc = system(commands);
if (0 != rc)
{
exists = file_dir_exists("./test-result/imagefile.iso.dir/test.jar");
FO_ASSERT_EQUAL(exists, 1); // existing
exists = file_dir_exists("./test-result/imagefile.iso.dir/test.jar.dir");
FO_ASSERT_EQUAL(exists, 0); // not existing
}
else
{
exists = file_dir_exists("./test-result/imagefile.iso.dir/TEST.JAR;1");
FO_ASSERT_EQUAL(exists, 1); // existing
}
}
}
/**
* @brief unpack deb file
*/
void testExtractAR4DebFile()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
FO_ASSERT_EQUAL(exists, 0); // not existing
MkDirs("./test-result/bind9-host_1%3a9.7.0.dfsg.P1-1_i386.deb.dir/");
Filename = "../test-data/testdata4unpack/bind9-host_1%3a9.7.0.dfsg.P1-1_i386.deb";
Result = ExtractAR(Filename, "./test-result/bind9-host_1%3a9.7.0.dfsg.P1-1_i386.deb.dir");
exists = file_dir_exists("./test-result/bind9-host_1%3a9.7.0.dfsg.P1-1_i386.deb.dir/data.tar.gz");
FO_ASSERT_EQUAL(exists, 1); // existing
FO_ASSERT_EQUAL(Result, 0); // Extract archieve library successfully
}
/** @brief unpack archive library file
**/
void testExtractAR4ArchiveLibraryFile()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
FO_ASSERT_EQUAL(exists, 0); // not existing
MkDirs("./test-result/libfossagent.a.dir/");
Filename = "../test-data/testdata4unpack/libfossagent.a";
Result = ExtractAR(Filename, "./test-result/libfossagent.a.dir");
exists = file_dir_exists("./test-result/libfossagent.a.dir/libfossagent.o");
FO_ASSERT_EQUAL(exists, 1); // existing
FO_ASSERT_EQUAL(Result, 0); // Extract archieve library successfully
}
/**
* @brief unpack disk image, ntfs
*/
void testExtractDisk4Ntfs()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
FO_ASSERT_EQUAL(exists, 0); // not existing
Filename = "../test-data/testdata4unpack/ntfstest-image";
MkDirs("./test-result/ntfstest-image.dir");
Result = ExtractDisk(Filename, "ntfs", "./test-result/ntfstest-image.dir");
exists = file_dir_exists("./test-result/ntfstest-image.dir/ununpack.c");
FO_ASSERT_EQUAL(Result, 0);
FO_ASSERT_EQUAL(exists, 1);
}
/**
* @brief unpack disk image, ext2, FStype is unknowed
*/
void testExtractDisk4Ext2FstypeUnknow()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
FO_ASSERT_EQUAL(exists, 0); // not existing
Filename = "../test-data/testdata4unpack/ext2test-image";
MkDirs("./test-result/ext2test-image.dir");
Result = ExtractDisk(Filename, "", "./test-result/ext2test-image.dir");
exists = file_dir_exists("./test-result/ext2test-image.dir/ununpack.c");
FO_ASSERT_EQUAL(Result, 1);
FO_ASSERT_EQUAL(exists, 0);
}
/**
* @brief unpack disk image, ext3
*/
void testExtractDisk4Ext3()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
FO_ASSERT_EQUAL(exists, 0); // not existing
Filename = "../test-data/testdata4unpack/ext3test-image";
MkDirs("./test-result/ext3test-image.dir");
Result = ExtractDisk(Filename, "ext", "./test-result/ext3test-image.dir");
exists = file_dir_exists("./test-result/ext3test-image.dir/libfossagent.a");
FO_ASSERT_EQUAL(Result, 0);
FO_ASSERT_EQUAL(exists, 1);
}
/**
* @brief test the command zcat, just testing if the commmand can run
*/
void testRunCommand4ZcatTesting()
{
deleteTmpFiles("./test-result/");
Cmd = "zcat";
CmdPre = "-q -l";
File = "../test-data/testdata4unpack/FileName.tar.Z";
CmdPost = ">/dev/null 2>&1";
Out = 0x0;
Where = 0x0;
Result = RunCommand(Cmd, CmdPre, File, CmdPost, Out, Where);
exists = file_dir_exists("./test-result/");
FO_ASSERT_EQUAL(exists, 0); // ./test-result/ is not existing
FO_ASSERT_EQUAL(Result, 0); // command could run
}
/**
* @brief test rpm file, the command is rmp2cpio
*/
void testRunCommand4Rpm1()
{
deleteTmpFiles("./test-result/");
Cmd = "rpm2cpio";
CmdPre = "";
File = "../test-data/testdata4unpack/fossology-1.2.0-1.el5.i386.rpm";
CmdPost = "> '%s' 2> /dev/null";
Out = "fossology-1.2.0-1.el5.i386.rpm.unpacked";
Where = "./test-result";
Result = RunCommand(Cmd, CmdPre, File, CmdPost, Out, Where);
exists = file_dir_exists("./test-result/fossology-1.2.0-1.el5.i386.rpm.unpacked");
FO_ASSERT_EQUAL(exists, 1); // existing
FO_ASSERT_EQUAL(Result, 0); // command could run
}
/**
* @brief test the command pdftotext
*/
void testRunCommand4Pdf()
{
deleteTmpFiles("./test-result/");
Cmd = "pdftotext";
CmdPre = "-htmlmeta";
File = "../test-data/testdata4unpack/israel.pdf";
CmdPost = "> '%s' 2>/dev/null";
Out = "israel.pdf.text";
Where = "./test-result";
Result = RunCommand(Cmd, CmdPre, File, CmdPost, Out, Where);
exists = file_dir_exists("./test-result/israel.pdf.text");
FO_ASSERT_EQUAL(exists, 1); // the file is existing
FO_ASSERT_EQUAL(Result, 0); // command could run
}
/**
* @brief test the command zcat, unpack via zcat
*/
void testRunCommand4Zcat()
{
deleteTmpFiles("./test-result/");
Cmd = "zcat";
CmdPre = "";
File = "../test-data/testdata4unpack/FileName.tar.Z";
CmdPost = "> '%s' 2>/dev/null";
Out = "FileName.tar.Z.unpacked";
Where = "./test-result";
Result = RunCommand(Cmd, CmdPre, File, CmdPost, Out, Where);
exists = file_dir_exists("./test-result/FileName.tar.Z.unpacked");
FO_ASSERT_EQUAL(exists, 1); // the file is existing
FO_ASSERT_EQUAL(Result, 0); // command could run
}
/**
* @brief test rpm file, the command is cpio
*/
void testRunCommand4Rpm2()
{
deleteTmpFiles("./test-result/");
testRunCommand4Rpm1();
Cmd = "cpio";
CmdPre = "--no-absolute-filenames -i -d <";
File = "./test-result/fossology-1.2.0-1.el5.i386.rpm.unpacked";
CmdPost = ">/dev/null 2>&1";
Out = "fossology-1.2.0-1.el5.i386.rpm.unpacked.dir";
Where = "./test-result/fossology-1.2.0-1.el5.i386.rpm.unpacked.dir";
Result = RunCommand(Cmd, CmdPre, File, CmdPost, Out, Where);
exists = file_dir_exists("./test-result/fossology-1.2.0-1.el5.i386.rpm.unpacked.dir/etc/fossology/RepPath.conf");
FO_ASSERT_EQUAL(exists, 1); // existing
FO_ASSERT_EQUAL(Result, 0); // command could run
}
/**
* @brief unpack debian source
*/
void testTraverseChild4DebianSourceFile()
{
deleteTmpFiles("./test-result/");
exists = file_dir_exists("./test-result/");
Filename = "../test-data/testdata4unpack/fcitx_3.6.2-1.dsc";
// MkDirs("./test-result/fcitx_3.6.2-1.dsc.dir/");
lstat(Filename, &Stat);
ContainerInfo CITemp;
memset(&CITemp,0,sizeof(ContainerInfo));
strcpy(CITemp.Source, Filename);
strcpy(CITemp.Partdir, NewDir);
strcpy(CITemp.Partname, "fcitx_3.6.2-1.dsc");
strcpy(CITemp.PartnameNew, "fcitx_3.6.2-1.dsc.dir");
ParentInfo PITemp = {27, 1287725739, 1287725739, 0};
CITemp.Stat = Stat;
CITemp.PI = PITemp;
CITemp.uploadtree_pk = 0;
CITemp.Artifact = 0;
CITemp.IsDir = 0;
CITemp.IsCompressed = 0;
CITemp.uploadtree_pk = 0;
CITemp.pfile_pk = 0;
CITemp.ufile_mode = 0;
/* test TraverseChild */
int Pid;
Pid = fork();
if (Pid == 0)
{
TraverseChild(Index, &CITemp, NewDir);
} else
{
ParentWait();
exists = file_dir_exists("./test-result/fcitx_3.6.2-1.dsc.dir/debian/README.Debian");
FO_ASSERT_EQUAL(exists, 1); // existing
}
}
int main (int argc, const char * argv[]){
struct timeval start, end;
gettimeofday(&start, NULL);
// general parameters
size_t maxSeqLen = 50000;
int seqType = Sequence::AMINO_ACIDS;
// parameter for the prefiltering
int kmerSize = 6;
int alphabetSize = 21;
size_t maxResListLen = 100;
int split = 0;
int skip = 0;
bool aaBiasCorrection = true;
float zscoreThr = 50.0f;
float sensitivity = 4.0;
// parameters for the alignment
double evalThr = 0.001;
double covThr = 0.8;
int maxAlnNum = 10;
std::string lastSeqDB = "";
std::string currentSeqDB = "";
std::string cluDB = "";
std::string outDB = "";
std::string tmpDir = "";
// get the path of the scoring matrix
char* mmdir = getenv ("MMDIR");
if (mmdir == 0){
std::cerr << "Please set the environment variable $MMDIR to your MMSEQS installation directory.\n";
exit(1);
}
std::string scoringMatrixFile(mmdir);
scoringMatrixFile = scoringMatrixFile + "/data/blosum62.out";
parseArgs(argc, argv, &lastSeqDB, ¤tSeqDB, &cluDB, &outDB, &tmpDir, &scoringMatrixFile, &maxSeqLen);
std::string lastSeqDBIndex = lastSeqDB + ".index";
std::string currentSeqDBIndex = currentSeqDB + ".index";
std::string cluDBIndex = cluDB + ".index";
std::string outDBIndex = outDB + ".index";
std::list<std::string>* tmpFiles = new std::list<std::string>();
std::string AIndex = tmpDir + "/A.index";
std::string BIndex = tmpDir + "/B.index";
tmpFiles->push_back(AIndex);
tmpFiles->push_back(BIndex);
std::string Brest_indexFile = tmpDir + "/Brest.index";
tmpFiles->push_back(Brest_indexFile);
std::string BB_clu = tmpDir + "/BB_clu";
std::string BB_clu_index = BB_clu + ".index";
tmpFiles->push_back(BB_clu);
tmpFiles->push_back(BB_clu_index);
std::cout << "////////////////////////////////////////////////////////////////////////\n";
std::cout << "/////// Init /////////////\n";
std::cout << "////////////////////////////////////////////////////////////////////////\n";
// extract three indexes:
// - A: last database version without deleted sequences
// - B: sequences which are new in the database
writeIndexes(AIndex, BIndex, lastSeqDBIndex, currentSeqDBIndex);
std::cout << "////////////////////////////////////////////////////////////////////////\n";
std::cout << "/////// Calculating B->A scores /////////////\n";
std::cout << "////////////////////////////////////////////////////////////////////////\n";
// calculate score for the updating
// B->A scores
std::string BA_base = runScoresCalculation(currentSeqDB, BIndex,
currentSeqDB, AIndex,
tmpDir,
scoringMatrixFile, maxSeqLen, seqType,
kmerSize, alphabetSize, maxResListLen, split, skip, aaBiasCorrection, zscoreThr, sensitivity,
evalThr, covThr, maxAlnNum, "BA", tmpFiles);
std::cout << "////////////////////////////////////////////////////////////////////////\n";
std::cout << "/////// Adding sequences to existing clusters /////////////\n";
std::cout << "////////////////////////////////////////////////////////////////////////\n";
// update the clustering
DBReader* currSeqDbr = new DBReader(currentSeqDB.c_str(), currentSeqDBIndex.c_str());
currSeqDbr->open(DBReader::NOSORT);
// data structures for the clustering
int seqDBSize = currSeqDbr->getSize();
unsigned int* id2rep = new unsigned int[seqDBSize];
char** rep2cluName = new char*[seqDBSize];
for (int i = 0; i < seqDBSize; i++)
rep2cluName[i] = new char[FFINDEX_MAX_ENTRY_NAME_LENTH];
cluster_t* clusters = new cluster_t[seqDBSize];
for (int i = 0; i < seqDBSize; i++){
clusters[i].clu_size = 0;
clusters[i].first = 0;
clusters[i].last = 0;
}
std::cout << "Read the existing clustering...\n";
// Read the existing clustering
readClustering(currSeqDbr, cluDB, id2rep, rep2cluName, clusters);
std::cout << "Append new sequences to the existing clustering...\n";
// append sequences from the new database to the existing clustering based on the B->A alignment scores
// write sequences without a match to a separate index (they will be clustered separately)
appendToClustering(currSeqDbr, BIndex, BA_base, id2rep, clusters, Brest_indexFile);
if (seqsWithoutMatches > 0){
std::cout << "////////////////////////////////////////////////////////////////////////\n";
std::cout << "/////// Calculating B->B scores /////////////\n";
std::cout << "////////////////////////////////////////////////////////////////////////\n";
// B->B scores
std::string BB_base = runScoresCalculation(currentSeqDB, Brest_indexFile,
currentSeqDB, Brest_indexFile,
tmpDir,
scoringMatrixFile, maxSeqLen, seqType,
kmerSize, alphabetSize, maxResListLen, split, skip, aaBiasCorrection, zscoreThr, sensitivity,
evalThr, covThr, maxAlnNum, "BB", tmpFiles);
std::cout << "////////////////////////////////////////////////////////////////////////\n";
std::cout << "/////// Appending new clusters /////////////\n";
std::cout << "////////////////////////////////////////////////////////////////////////\n";
std::cout << "Cluster new sequences without a match to the existing clusters...\n";
// cluster sequences without a match to the existing clusters separately
// use the index generated in the previous step
Clustering* clu = new Clustering(currentSeqDB, currentSeqDBIndex,
BB_base, BB_base + ".index",
BB_clu, BB_clu_index,
0.0, 0, maxResListLen);
clu->run(Clustering::SET_COVER);
std::cout << "Append generated clusters to the complete clustering...\n";
// append B->B clusters to the clustering
newClus = readClustering(currSeqDbr, BB_clu, id2rep, rep2cluName, clusters);
}
// write new clustering
std::cout << "Write clustering results...\n";
writeResults(clusters, rep2cluName, currSeqDbr, seqDBSize, outDB);
std::cout << "done.\n";
currSeqDbr->close();
std::cout << "////////////////////////////////////////////////////////////////////////\n";
std::cout << "/////// Statistics ////////\n";
std::cout << "////////////////////////////////////////////////////////////////////////\n";
std::cout << "\nPrevios database version: " << oldDBSize << " entries.\n";
std::cout << "New database vesion : " << newDBSize << " entries.\n";
std::cout << deletedSeqs << " entries were deleted,\n";
std::cout << newSeqs << " entries are new,\n";
std::cout << sharedSeqs << " entries are shared.\n\n";
std::cout << seqsWithMatches << " new sequences had matches to the previous database version.\n";
std::cout << "Remaining " << seqsWithoutMatches << " were grouped into " << newClus << " new clusters.\n";
gettimeofday(&end, NULL);
int sec = end.tv_sec - start.tv_sec;
std::cout << "\nTime for updating: " << (sec / 3600) << " h " << (sec % 3600 / 60) << " m " << (sec % 60) << "s\n\n";
deleteTmpFiles(tmpFiles);
delete tmpFiles;
}