/**
* \brief Given an upload ID, delete it.
*
* \param uploadId the upload id
* \param userId
* \param userPerm permission level the user has
*
* \return 0: yes, can is deleted;
* 1: can not be deleted;
* -1: failure;
* -2: does not exist
*/
int deleteUpload (long uploadId, int userId, int userPerm)
{
char *S;
int Row,maxRow;
char tempTable[256];
PGresult *result, *pfileResult;
char SQL[MAXSQL], desc[myBUFSIZ];
int permission_upload = check_write_permission_upload(uploadId, userId, userPerm);
if(0 != permission_upload) {
return permission_upload;
}
snprintf(tempTable,sizeof(tempTable),"DelUp_%ld_pfile",uploadId);
snprintf(SQL,MAXSQL,"DROP TABLE IF EXISTS %s;",tempTable);
PQexecCheckClear(NULL, SQL, __FILE__, __LINE__);
snprintf(desc, myBUFSIZ, "Deleting upload %ld",uploadId);
PQexecCheckClear(desc, "SET statement_timeout = 0;", __FILE__, __LINE__);
PQexecCheckClear(NULL, "BEGIN;", __FILE__, __LINE__);
/* Delete everything that impacts the UI */
if (!Test) {
/* The UI depends on uploadtree and folders for navigation.
Delete them now to block timeouts from the UI. */
PQexecCheckClear(NULL, "COMMIT;", __FILE__, __LINE__);
}
/* Begin complicated stuff */
/* Get the list of pfiles to delete */
/* These are all pfiles in the upload_fk that only appear once. */
snprintf(SQL,MAXSQL,"SELECT DISTINCT pfile_pk,pfile_sha1 || '.' || pfile_md5 || '.' || pfile_size AS pfile INTO %s FROM uploadtree INNER JOIN pfile ON upload_fk = %ld AND pfile_fk = pfile_pk;",tempTable,uploadId);
PQexecCheckClear("Getting list of pfiles to delete", SQL, __FILE__, __LINE__);
/* Remove pfiles which are reused by other uploads */
snprintf(SQL, MAXSQL, "DELETE FROM %s WHERE pfile_pk IN (SELECT pfile_pk FROM %s INNER JOIN uploadtree ON pfile_pk = pfile_fk WHERE upload_fk != %ld)", tempTable, tempTable, uploadId);
PQexecCheckClear(NULL, SQL, __FILE__, __LINE__);
if (Verbose) {
snprintf(SQL,MAXSQL,"SELECT COUNT(*) FROM %s;",tempTable);
result = PQexec(pgConn, SQL);
if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) {
return -1;
}
printf("# Created pfile table %s with %ld entries\n", tempTable, atol(PQgetvalue(result,0,0)));
PQclear(result);
}
/* Now to delete the actual pfiles from the repository before remove the DB. */
/* Get the file listing -- needed for deleting pfiles from the repository. */
snprintf(SQL,MAXSQL,"SELECT * FROM %s ORDER BY pfile_pk;",tempTable);
pfileResult = PQexec(pgConn, SQL);
if (fo_checkPQresult(pgConn, pfileResult, SQL, __FILE__, __LINE__)) {
return -1;
}
if (Test <= 1) {
maxRow = PQntuples(pfileResult);
for(Row=0; Row<maxRow; Row++) {
S = PQgetvalue(pfileResult,Row,1); /* sha1.md5.len */
if (fo_RepExist("files",S)) {
if (Test) {
printf("TEST: Delete %s %s\n","files",S);
} else {
fo_RepRemove("files",S);
}
}
if (fo_RepExist("gold",S)) {
if (Test) {
printf("TEST: Delete %s %s\n","gold",S);
} else {
fo_RepRemove("gold",S);
}
}
fo_scheduler_heart(1);
}
}
PQclear(pfileResult);
/*
This begins the slow part that locks the DB.
The problem is, we don't want to lock a critical row,
otherwise the scheduler will lock and/or fail.
*/
if (!Test) {
PQexecCheckClear(NULL, "BEGIN;", __FILE__, __LINE__);
}
/* Delete the upload from the folder-contents table */
snprintf(SQL,MAXSQL,"DELETE FROM foldercontents WHERE (foldercontents_mode & 2) != 0 AND child_id = %ld;",uploadId);
PQexecCheckClear("Deleting foldercontents", SQL, __FILE__, __LINE__);
/* Deleting the actual upload contents*/
/* Delete the bucket_container record as it can't be cascade delete with upload table */
snprintf(SQL,MAXSQL,"DELETE FROM bucket_container USING uploadtree WHERE uploadtree_fk = uploadtree_pk AND upload_fk = %ld;",uploadId);
PQexecCheckClear("Deleting bucket_container", SQL, __FILE__, __LINE__);
/* Delete the tag_uploadtree record as it can't be cascade delete with upload table */
snprintf(SQL,MAXSQL,"DELETE FROM tag_uploadtree USING uploadtree WHERE uploadtree_fk = uploadtree_pk AND upload_fk = %ld;",uploadId);
PQexecCheckClear("Deleting tag_uploadtree", SQL, __FILE__, __LINE__);
/* Delete uploadtree_nnn table */
char uploadtree_tablename[1024];
snprintf(SQL,MAXSQL,"SELECT uploadtree_tablename FROM upload WHERE upload_pk = %ld;",uploadId);
result = PQexec(pgConn, SQL);
if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) {
return -1;
}
if (PQntuples(result)) {
strcpy(uploadtree_tablename, PQgetvalue(result, 0, 0));
PQclear(result);
if (strcasecmp(uploadtree_tablename,"uploadtree_a")) {
snprintf(SQL,MAXSQL,"DROP TABLE %s;", uploadtree_tablename);
PQexecCheckClear(NULL, SQL, __FILE__, __LINE__);
}
}
printfInCaseOfVerbosity("Deleting license decisions for upload %ld\n",uploadId);
/* delete from clearing_decision_event table. */
snprintf(SQL, MAXSQL, "DELETE FROM clearing_decision_event USING clearing_event WHERE clearing_decision_event.clearing_event_fk = clearing_event.clearing_event_pk AND clearing_event.uploadtree_fk IN (SELECT uploadtree_pk FROM uploadtree INNER JOIN %s ON uploadtree.pfile_fk = %s.pfile_pk WHERE upload_fk = %ld);", tempTable, tempTable, uploadId);
PQexecCheckClear("Deleting from clearing_decision_event", SQL, __FILE__, __LINE__);
/* delete from clearing_event table. */
snprintf(SQL, MAXSQL, "DELETE FROM clearing_event WHERE uploadtree_fk IN (SELECT uploadtree_pk FROM uploadtree INNER JOIN %s ON uploadtree.pfile_fk = %s.pfile_pk WHERE upload_fk = %ld);", tempTable, tempTable, uploadId);
PQexecCheckClear("Deleting from clearing_event", SQL, __FILE__, __LINE__);
/* delete from uploadtree table. */
snprintf(SQL, MAXSQL, "DELETE FROM uploadtree WHERE upload_fk = %ld;", uploadId);
PQexecCheckClear("Deleting from uploadtree", SQL, __FILE__, __LINE__);
/* delete from pfile is SLOW due to constraint checking. Do it separately. */
snprintf(SQL,MAXSQL,"DELETE FROM pfile USING %s WHERE pfile.pfile_pk = %s.pfile_pk;",tempTable,tempTable);
PQexecCheckClear("Deleting from pfile", SQL, __FILE__, __LINE__);
snprintf(SQL,MAXSQL,"DROP TABLE %s;",tempTable);
PQexecCheckClear(NULL, SQL, __FILE__, __LINE__);
PQexecCheckClear(NULL, "SET statement_timeout = 120000;", __FILE__, __LINE__);
printfInCaseOfVerbosity("Deleted upload %ld from DB, now doing repository.\n",uploadId);
if (Test) {
PQexecCheckClear(NULL, "ROLLBACK;", __FILE__, __LINE__);
} else {
PQexecCheckClear(NULL, "COMMIT;", __FILE__, __LINE__);
}
printfInCaseOfVerbosity("Deleted upload %ld\n",uploadId);
return 0; /* success */
} /* deleteUpload() */
/**
* @brief perform the analysis of a given list of files
*
* loops over the file_list given and performs the analysis of all the files,
* when finished simply check if the results should be entered into the
* database, this is indicated by the second element of the pair not being NULL
*
* @param pgConn the connection to the database
* @param copy the copyright instance to use to perform the analysis
* @param current_file the file and the pfile_pk that is currently being analyzed
* @param agent_pk the primary key for this agent, use to enter info into the database
* @param mout a logging file to used for debugging
* @param type report_type binary number xyz, 0x1(bit z): statement, 0x2(bit y): url, 0x4(bit x): email
*/
void perform_analysis(PGconn* pgConn, copyright copy, pair current_file, long agent_pk, FILE* mout, int report_type)
{
/* locals */
char sql[2048]; // buffer to hold the sql commands
char buf[2048]; // buffer to hold string that have been escaped for sql
char hash[256]; // holds the hash of the copyright string for entry into database
char* file_name; // holds the name of the file to open
copyright_iterator finds; // an iterator to access the copyrights
FILE* input_fp; // the file that will be analyzed
/* initialize memory */
memset(sql, 0, sizeof(sql));
file_name = NULL;
finds = NULL;
input_fp = NULL;
/* find the correct path to the file */
if(*(int*)pair_second(current_file) >= 0)
{
file_name = fo_RepMkPath("files", (char*)pair_first(current_file));
}
else
{
file_name = (char*)pair_first(current_file);
}
/* attempt to open the file */
input_fp = fopen(file_name, "rb");
if(!input_fp)
{
fprintf(cerr, "ERROR %s.%d Failure to open file %s\n", __FILE__, __LINE__, file_name);
fprintf(cerr, "ERROR %s\n", strerror(errno));
fflush(cerr);
copyright_clear(copy);
return;
}
/* only free temp if running as an agent */
if(*(int*)pair_second(current_file) >= 0)
{
free(file_name);
}
/* perform the actual analysis */
copyright_analyze(copy, input_fp, report_type);
/* if running command line, print file name */
if(*(int*)pair_second(current_file) < 0)
{
fprintf(cout, "%s ::\n", (char*)pair_first(current_file));
}
/* loop across the found copyrights */
if(copyright_size(copy) > 0)
{
for(finds = copyright_begin(copy); finds != copyright_end(copy); finds++)
{
copy_entry entry = *finds;
if(verbose)
{
fprintf(mout, "=== %s ==============================================\n",
(char*)pair_first(current_file));
fprintf(mout, "DICT: %s\nNAME: %s\nTEXT[%s]\n",
copy_entry_dict(entry),
copy_entry_name(entry),
copy_entry_text(entry));
}
if(*(int*)pair_second(current_file) >= 0)
{
/* ensure legal sql */
escape_string(pgConn, buf, copy_entry_text(entry), sizeof(buf));
/* get the hash for the string */
sprintf(hash, "0x%lx", hash_string(copy_entry_text(entry)));
/* place the copyright in the table */
memset(sql, '\0', sizeof(sql));
snprintf(sql, sizeof(sql), "%ld\t%d\t%d\t%d\t%s\t%s\t%s\n",
agent_pk,
*(int*)pair_second(current_file),
copy_entry_start(entry),
copy_entry_end(entry),
buf,
hash,
copy_entry_type(entry));
fo_sqlCopyAdd(sqlcpy, sql);
}
else
{
fprintf(cout, "\t[%d:%d:%s] '%s'",
copy_entry_start(entry), copy_entry_end(entry),
copy_entry_type(entry), copy_entry_text(entry));
if(copy_entry_text(entry)[strlen(copy_entry_text(entry)) - 1] != '\n')
{
fprintf(cout, "\n");
}
}
}
}
fclose(input_fp);
fo_scheduler_heart(1);
}
/**
* \brief Get the mimetype for a package
* \param argc the number of command line arguments
* \param argv the command line arguments
* \return 0 on a successful program execution
*/
int main(int argc, char *argv[])
{
int arg;
char *Parm = NULL;
char *Path = NULL;
int c;
char *agent_desc = "Determines mimetype for each file";
int pfile_count = 0;
int Agent_pk;
int ars_pk = 0;
int upload_pk = 0; // the upload primary key
int user_pk = 0;
char *AgentARSName = "mimetype_ars";
int rv;
PGresult *result;
char sqlbuf[1024];
int CmdlineFlag = 0; ///< run from command line flag, 1 yes, 0 not
char *COMMIT_HASH;
char *VERSION;
char agent_rev[MAXCMD];
/* initialize the scheduler connection */
fo_scheduler_connect(&argc, argv, &pgConn);
/* Process command-line */
while((c = getopt(argc,argv,"iCc:hvV")) != -1)
{
switch(c)
{
case 'i':
PQfinish(pgConn);
return(0);
case 'c':
/* do nothing with this option */
break;
case 'C':
CmdlineFlag = 1;
break;
case 'v':
agent_verbose++;
break;
case 'V':
printf("%s", BuildVersion);
PQfinish(pgConn);
return(0);
default:
Usage(argv[0]);
PQfinish(pgConn);
exit(-1);
}
}
COMMIT_HASH = fo_sysconfig("mimetype", "COMMIT_HASH");
VERSION = fo_sysconfig("mimetype", "VERSION");
sprintf(agent_rev, "%s.%s", VERSION, COMMIT_HASH);
/* Get the Agent Key from the DB */
Agent_pk = fo_GetAgentKey(pgConn, basename(argv[0]), 0, agent_rev, agent_desc);
FMimetype = fopen("/etc/mime.types","rb");
if (!FMimetype)
{
LOG_WARNING("Unable to open /etc/mime.types\n");
}
MagicCookie = magic_open(MAGIC_PRESERVE_ATIME|MAGIC_MIME);
if (MagicCookie == NULL)
{
LOG_FATAL("Failed to initialize magic cookie\n");
PQfinish(pgConn);
exit(-1);
}
if (magic_load(MagicCookie,NULL) != 0)
{
LOG_FATAL("Failed to load magic file: UnMagic\n");
PQfinish(pgConn);
exit(-1);
}
/* Run from the command-line (for testing) */
for(arg=optind; arg < argc; arg++)
{
Akey = -1;
memset(A,'\0',sizeof(A));
strncpy(A,argv[arg],sizeof(A));
DBCheckMime(A);
}
/* Run from scheduler! */
if (0 == CmdlineFlag)
{
user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */
while(fo_scheduler_next())
{
/* get piece of information, including upload_pk, others */
Parm = fo_scheduler_current();
if (Parm && Parm[0])
{
upload_pk = atoi(Parm);
/* Check Permissions */
if (GetUploadPerm(pgConn, upload_pk, user_pk) < PERM_WRITE)
{
LOG_ERROR("You have no update permissions on upload %d", upload_pk);
continue;
}
/* does ars table exist?
* If not, create it.
*/
rv = fo_tableExists(pgConn, AgentARSName);
if (!rv)
{
rv = fo_CreateARSTable(pgConn, AgentARSName);
if (!rv) return(0);
}
/* check ars table if this is duplicate request*/
memset(sqlbuf, 0, sizeof(sqlbuf));
snprintf(sqlbuf, sizeof(sqlbuf),
"select ars_pk from mimetype_ars,agent \
where agent_pk=agent_fk and ars_success=true \
and upload_fk='%d' and agent_fk='%d'",
upload_pk, Agent_pk);
result = PQexec(pgConn, sqlbuf);
if (fo_checkPQresult(pgConn, result, sqlbuf, __FILE__, __LINE__)) exit(-1);
if (PQntuples(result) > 0)
{
PQclear(result);
LOG_WARNING("Ignoring requested mimetype analysis of upload %d - Results are already in database.\n",upload_pk);
continue;
}
PQclear(result);
/* Record analysis start in mimetype_ars, the mimetype audit trail. */
ars_pk = fo_WriteARS(pgConn, ars_pk, upload_pk, Agent_pk, AgentARSName, 0, 0);
/* get all pfile ids on a upload record */
memset(sqlbuf, 0, sizeof(sqlbuf));
snprintf(sqlbuf, sizeof(sqlbuf), "SELECT DISTINCT(pfile_pk) as Akey, pfile_sha1 || '.' || pfile_md5 || '.' || pfile_size AS A FROM uploadtree, pfile WHERE uploadtree.pfile_fk = pfile.pfile_pk AND pfile_mimetypefk is NULL AND upload_fk = '%d';", upload_pk);
result = PQexec(pgConn, sqlbuf);
if (fo_checkPQresult(pgConn, result, sqlbuf, __FILE__, __LINE__)) exit(-1);
pfile_count = PQntuples(result);
int i;
for(i=0; i < pfile_count; i++)
{
Akey = atoi(PQgetvalue(result, i, 0));
strncpy(A, PQgetvalue(result, i, 1), sizeof(A));
if (Akey <= 0 || A[0]=='\0')
{
printf("ERROR: Data is in an unknown format.\n");
PQfinish(pgConn);
exit(-1);
}
/* Process the repository file */
/* Find the path */
Path = fo_RepMkPath("files",A);
if (Path && fo_RepExist("files",A))
{
/* Get the mimetype! */
DBCheckMime(Path);
}
else
{
printf("ERROR pfile %d Unable to process.\n",Akey);
printf("LOG pfile %d File '%s' not found.\n",Akey,A);
PQfinish(pgConn);
exit(-1);
}
/* Clean up Path memory */
if(Path)
{
free(Path);
Path = NULL;
}
fo_scheduler_heart(1);
}
PQclear(result);
/* Record analysis success in mimetype_ars. */
if (ars_pk) fo_WriteARS(pgConn, ars_pk, upload_pk, Agent_pk, AgentARSName, 0, 1);
}
}
} /* if run from scheduler */
/* Clean up */
if (FMimetype) fclose(FMimetype);
magic_close(MagicCookie);
if (DBMime) PQclear(DBMime);
if (pgConn) PQfinish(pgConn);
/* after cleaning up agent, disconnect from the scheduler, this doesn't return */
fo_scheduler_disconnect(0);
return(0);
} /* main() */
int main (int argc, char *argv[])
{
int arg;
char *Parm = NULL;
char *TempFileDir=NULL;
int c;
int InitFlag=0;
int CmdlineFlag = 0; /** run from command line flag, 1 yes, 0 not */
int user_pk;
char *agent_desc = "Network downloader. Uses wget(1).";
memset(GlobalTempFile,'\0',MAXCMD);
memset(GlobalURL,'\0',MAXCMD);
memset(GlobalParam,'\0',MAXCMD);
memset(GlobalType,'\0',MAXCMD);
GlobalUploadKey = -1;
int upload_pk = 0; // the upload primary key
//int Agent_pk;
char *SVN_REV;
char *VERSION;
char agent_rev[MAXCMD];
/* open the connection to the scheduler and configuration */
fo_scheduler_connect(&argc, argv, &pgConn);
/* Process command-line */
while((c = getopt(argc,argv,"d:Gg:ik:A:R:l:Cc:Vvh")) != -1)
{
switch(c)
{
case 'd':
TempFileDir = PathCheck(optarg);
break;
case 'g':
{
struct group *SG;
SG = getgrnam(optarg);
if (SG) ForceGroup = SG->gr_gid;
}
break;
case 'G':
GlobalImportGold=0;
break;
case 'i':
InitFlag=1;
break;
case 'k':
GlobalUploadKey = atol(optarg);
if (!GlobalTempFile[0])
strcpy(GlobalTempFile,"wget.default_download");
break;
case 'A':
sprintf(GlobalParam, "%s -A %s ",GlobalParam, optarg);
break;
case 'R':
sprintf(GlobalParam, "%s -R %s ",GlobalParam, optarg);
break;
case 'l':
sprintf(GlobalParam, "%s -l %s ",GlobalParam, optarg);
break;
case 'c':
break; /* handled by fo_scheduler_connect() */
case 'C':
CmdlineFlag = 1;
break;
case 'v':
break;
case 'V':
printf("%s", BuildVersion);
SafeExit(0);
default:
Usage(argv[0]);
SafeExit(-1);
}
}
if (argc - optind > 1)
{
Usage(argv[0]);
SafeExit(-1);
}
/* When initializing the DB, don't do anything else */
if (InitFlag)
{
if (pgConn) PQfinish(pgConn);
SafeExit(0);
}
SVN_REV = fo_sysconfig("wget_agent", "SVN_REV");
VERSION = fo_sysconfig("wget_agent", "VERSION");
sprintf(agent_rev, "%s.%s", VERSION, SVN_REV);
/* Get the Agent Key from the DB */
fo_GetAgentKey(pgConn, basename(argv[0]), GlobalUploadKey, agent_rev, agent_desc);
/** get proxy */
GetProxy();
/* Run from the command-line (for testing) */
for(arg=optind; arg < argc; arg++)
{
memset(GlobalURL,'\0',sizeof(GlobalURL));
strncpy(GlobalURL,argv[arg],sizeof(GlobalURL));
/* If the file contains "://" then assume it is a URL.
Else, assume it is a file. */
LOG_VERBOSE0("Command-line: %s",GlobalURL);
if (strstr(GlobalURL,"://"))
{
fo_scheduler_heart(1);
LOG_VERBOSE0("It's a URL");
if (GetURL(GlobalTempFile,GlobalURL,TempFileDir) != 0)
{
LOG_FATAL("Download of %s failed.",GlobalURL);
SafeExit(21);
}
if (GlobalUploadKey != -1) {
DBLoadGold();
}
unlink(GlobalTempFile);
}
else /* must be a file */
{
LOG_VERBOSE0("It's a file -- GlobalUploadKey = %ld",GlobalUploadKey);
if (GlobalUploadKey != -1)
{
memcpy(GlobalTempFile,GlobalURL,MAXCMD);
DBLoadGold();
}
}
}
/* Run from scheduler! */
if (0 == CmdlineFlag)
{
user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */
while(fo_scheduler_next())
{
Parm = fo_scheduler_current(); /* get piece of information, including upload_pk, downloadfile url, and parameters */
if (Parm && Parm[0])
{
fo_scheduler_heart(1);
/* set globals: uploadpk, downloadfile url, parameters */
SetEnv(Parm,TempFileDir);
upload_pk = GlobalUploadKey;
/* Check Permissions */
if (GetUploadPerm(pgConn, upload_pk, user_pk) < PERM_WRITE)
{
LOG_ERROR("You have no update permissions on upload %d", upload_pk);
continue;
}
char TempDir[MAXCMD];
memset(TempDir,'\0',MAXCMD);
snprintf(TempDir, MAXCMD-1, "%s/wget", TempFileDir); // /var/local/lib/fossology/agents/wget
struct stat Status;
if (GlobalType[0])
{
if (GetVersionControl() == 0)
{
DBLoadGold();
unlink(GlobalTempFile);
}
else
{
LOG_FATAL("upload %ld File retrieval failed: uploadpk=%ld tempfile=%s URL=%s Type=%s",
GlobalUploadKey,GlobalUploadKey,GlobalTempFile,GlobalURL, GlobalType);
SafeExit(23);
}
}
else if (strstr(GlobalURL, "*") || stat(GlobalURL, &Status) == 0)
{
if (!Archivefs(GlobalURL, GlobalTempFile, TempFileDir, Status))
{
LOG_FATAL("Failed to archieve. GlobalURL, GlobalTempFile, TempFileDir are: %s, %s, %s, "
"Mode is: %lo (octal)\n", GlobalURL, GlobalTempFile, TempFileDir, (unsigned long) Status.st_mode);
SafeExit(50);
}
DBLoadGold();
unlink(GlobalTempFile);
}
else
{
if (GetURL(GlobalTempFile,GlobalURL,TempDir) == 0)
{
DBLoadGold();
unlink(GlobalTempFile);
}
else
{
LOG_FATAL("upload %ld File retrieval failed: uploadpk=%ld tempfile=%s URL=%s",
GlobalUploadKey,GlobalUploadKey,GlobalTempFile,GlobalURL);
SafeExit(22);
}
}
}
}
} /* if run from scheduler */
SafeExit(0);
exit(0); /* to prevent compiler warning */
} /* main() */
