/** * \brief Given an upload ID, delete it. * * \param uploadId the upload id * \param userId * \param userPerm permission level the user has * * \return 0: yes, can is deleted; * 1: can not be deleted; * -1: failure; * -2: does not exist */ int deleteUpload (long uploadId, int userId, int userPerm) { char *S; int Row,maxRow; char tempTable[256]; PGresult *result, *pfileResult; char SQL[MAXSQL], desc[myBUFSIZ]; int permission_upload = check_write_permission_upload(uploadId, userId, userPerm); if(0 != permission_upload) { return permission_upload; } snprintf(tempTable,sizeof(tempTable),"DelUp_%ld_pfile",uploadId); snprintf(SQL,MAXSQL,"DROP TABLE IF EXISTS %s;",tempTable); PQexecCheckClear(NULL, SQL, __FILE__, __LINE__); snprintf(desc, myBUFSIZ, "Deleting upload %ld",uploadId); PQexecCheckClear(desc, "SET statement_timeout = 0;", __FILE__, __LINE__); PQexecCheckClear(NULL, "BEGIN;", __FILE__, __LINE__); /* Delete everything that impacts the UI */ if (!Test) { /* The UI depends on uploadtree and folders for navigation. Delete them now to block timeouts from the UI. */ PQexecCheckClear(NULL, "COMMIT;", __FILE__, __LINE__); } /* Begin complicated stuff */ /* Get the list of pfiles to delete */ /* These are all pfiles in the upload_fk that only appear once. */ snprintf(SQL,MAXSQL,"SELECT DISTINCT pfile_pk,pfile_sha1 || '.' || pfile_md5 || '.' || pfile_size AS pfile INTO %s FROM uploadtree INNER JOIN pfile ON upload_fk = %ld AND pfile_fk = pfile_pk;",tempTable,uploadId); PQexecCheckClear("Getting list of pfiles to delete", SQL, __FILE__, __LINE__); /* Remove pfiles which are reused by other uploads */ snprintf(SQL, MAXSQL, "DELETE FROM %s WHERE pfile_pk IN (SELECT pfile_pk FROM %s INNER JOIN uploadtree ON pfile_pk = pfile_fk WHERE upload_fk != %ld)", tempTable, tempTable, uploadId); PQexecCheckClear(NULL, SQL, __FILE__, __LINE__); if (Verbose) { snprintf(SQL,MAXSQL,"SELECT COUNT(*) FROM %s;",tempTable); result = PQexec(pgConn, SQL); if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) { return -1; } printf("# Created pfile table %s with %ld entries\n", tempTable, atol(PQgetvalue(result,0,0))); PQclear(result); } /* Now to delete the actual pfiles from the repository before remove the DB. */ /* Get the file listing -- needed for deleting pfiles from the repository. */ snprintf(SQL,MAXSQL,"SELECT * FROM %s ORDER BY pfile_pk;",tempTable); pfileResult = PQexec(pgConn, SQL); if (fo_checkPQresult(pgConn, pfileResult, SQL, __FILE__, __LINE__)) { return -1; } if (Test <= 1) { maxRow = PQntuples(pfileResult); for(Row=0; Row<maxRow; Row++) { S = PQgetvalue(pfileResult,Row,1); /* sha1.md5.len */ if (fo_RepExist("files",S)) { if (Test) { printf("TEST: Delete %s %s\n","files",S); } else { fo_RepRemove("files",S); } } if (fo_RepExist("gold",S)) { if (Test) { printf("TEST: Delete %s %s\n","gold",S); } else { fo_RepRemove("gold",S); } } fo_scheduler_heart(1); } } PQclear(pfileResult); /* This begins the slow part that locks the DB. The problem is, we don't want to lock a critical row, otherwise the scheduler will lock and/or fail. */ if (!Test) { PQexecCheckClear(NULL, "BEGIN;", __FILE__, __LINE__); } /* Delete the upload from the folder-contents table */ snprintf(SQL,MAXSQL,"DELETE FROM foldercontents WHERE (foldercontents_mode & 2) != 0 AND child_id = %ld;",uploadId); PQexecCheckClear("Deleting foldercontents", SQL, __FILE__, __LINE__); /* Deleting the actual upload contents*/ /* Delete the bucket_container record as it can't be cascade delete with upload table */ snprintf(SQL,MAXSQL,"DELETE FROM bucket_container USING uploadtree WHERE uploadtree_fk = uploadtree_pk AND upload_fk = %ld;",uploadId); PQexecCheckClear("Deleting bucket_container", SQL, __FILE__, __LINE__); /* Delete the tag_uploadtree record as it can't be cascade delete with upload table */ snprintf(SQL,MAXSQL,"DELETE FROM tag_uploadtree USING uploadtree WHERE uploadtree_fk = uploadtree_pk AND upload_fk = %ld;",uploadId); PQexecCheckClear("Deleting tag_uploadtree", SQL, __FILE__, __LINE__); /* Delete uploadtree_nnn table */ char uploadtree_tablename[1024]; snprintf(SQL,MAXSQL,"SELECT uploadtree_tablename FROM upload WHERE upload_pk = %ld;",uploadId); result = PQexec(pgConn, SQL); if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) { return -1; } if (PQntuples(result)) { strcpy(uploadtree_tablename, PQgetvalue(result, 0, 0)); PQclear(result); if (strcasecmp(uploadtree_tablename,"uploadtree_a")) { snprintf(SQL,MAXSQL,"DROP TABLE %s;", uploadtree_tablename); PQexecCheckClear(NULL, SQL, __FILE__, __LINE__); } } printfInCaseOfVerbosity("Deleting license decisions for upload %ld\n",uploadId); /* delete from clearing_decision_event table. */ snprintf(SQL, MAXSQL, "DELETE FROM clearing_decision_event USING clearing_event WHERE clearing_decision_event.clearing_event_fk = clearing_event.clearing_event_pk AND clearing_event.uploadtree_fk IN (SELECT uploadtree_pk FROM uploadtree INNER JOIN %s ON uploadtree.pfile_fk = %s.pfile_pk WHERE upload_fk = %ld);", tempTable, tempTable, uploadId); PQexecCheckClear("Deleting from clearing_decision_event", SQL, __FILE__, __LINE__); /* delete from clearing_event table. */ snprintf(SQL, MAXSQL, "DELETE FROM clearing_event WHERE uploadtree_fk IN (SELECT uploadtree_pk FROM uploadtree INNER JOIN %s ON uploadtree.pfile_fk = %s.pfile_pk WHERE upload_fk = %ld);", tempTable, tempTable, uploadId); PQexecCheckClear("Deleting from clearing_event", SQL, __FILE__, __LINE__); /* delete from uploadtree table. */ snprintf(SQL, MAXSQL, "DELETE FROM uploadtree WHERE upload_fk = %ld;", uploadId); PQexecCheckClear("Deleting from uploadtree", SQL, __FILE__, __LINE__); /* delete from pfile is SLOW due to constraint checking. Do it separately. */ snprintf(SQL,MAXSQL,"DELETE FROM pfile USING %s WHERE pfile.pfile_pk = %s.pfile_pk;",tempTable,tempTable); PQexecCheckClear("Deleting from pfile", SQL, __FILE__, __LINE__); snprintf(SQL,MAXSQL,"DROP TABLE %s;",tempTable); PQexecCheckClear(NULL, SQL, __FILE__, __LINE__); PQexecCheckClear(NULL, "SET statement_timeout = 120000;", __FILE__, __LINE__); printfInCaseOfVerbosity("Deleted upload %ld from DB, now doing repository.\n",uploadId); if (Test) { PQexecCheckClear(NULL, "ROLLBACK;", __FILE__, __LINE__); } else { PQexecCheckClear(NULL, "COMMIT;", __FILE__, __LINE__); } printfInCaseOfVerbosity("Deleted upload %ld\n",uploadId); return 0; /* success */ } /* deleteUpload() */
/** * @brief perform the analysis of a given list of files * * loops over the file_list given and performs the analysis of all the files, * when finished simply check if the results should be entered into the * database, this is indicated by the second element of the pair not being NULL * * @param pgConn the connection to the database * @param copy the copyright instance to use to perform the analysis * @param current_file the file and the pfile_pk that is currently being analyzed * @param agent_pk the primary key for this agent, use to enter info into the database * @param mout a logging file to used for debugging * @param type report_type binary number xyz, 0x1(bit z): statement, 0x2(bit y): url, 0x4(bit x): email */ void perform_analysis(PGconn* pgConn, copyright copy, pair current_file, long agent_pk, FILE* mout, int report_type) { /* locals */ char sql[2048]; // buffer to hold the sql commands char buf[2048]; // buffer to hold string that have been escaped for sql char hash[256]; // holds the hash of the copyright string for entry into database char* file_name; // holds the name of the file to open copyright_iterator finds; // an iterator to access the copyrights FILE* input_fp; // the file that will be analyzed /* initialize memory */ memset(sql, 0, sizeof(sql)); file_name = NULL; finds = NULL; input_fp = NULL; /* find the correct path to the file */ if(*(int*)pair_second(current_file) >= 0) { file_name = fo_RepMkPath("files", (char*)pair_first(current_file)); } else { file_name = (char*)pair_first(current_file); } /* attempt to open the file */ input_fp = fopen(file_name, "rb"); if(!input_fp) { fprintf(cerr, "ERROR %s.%d Failure to open file %s\n", __FILE__, __LINE__, file_name); fprintf(cerr, "ERROR %s\n", strerror(errno)); fflush(cerr); copyright_clear(copy); return; } /* only free temp if running as an agent */ if(*(int*)pair_second(current_file) >= 0) { free(file_name); } /* perform the actual analysis */ copyright_analyze(copy, input_fp, report_type); /* if running command line, print file name */ if(*(int*)pair_second(current_file) < 0) { fprintf(cout, "%s ::\n", (char*)pair_first(current_file)); } /* loop across the found copyrights */ if(copyright_size(copy) > 0) { for(finds = copyright_begin(copy); finds != copyright_end(copy); finds++) { copy_entry entry = *finds; if(verbose) { fprintf(mout, "=== %s ==============================================\n", (char*)pair_first(current_file)); fprintf(mout, "DICT: %s\nNAME: %s\nTEXT[%s]\n", copy_entry_dict(entry), copy_entry_name(entry), copy_entry_text(entry)); } if(*(int*)pair_second(current_file) >= 0) { /* ensure legal sql */ escape_string(pgConn, buf, copy_entry_text(entry), sizeof(buf)); /* get the hash for the string */ sprintf(hash, "0x%lx", hash_string(copy_entry_text(entry))); /* place the copyright in the table */ memset(sql, '\0', sizeof(sql)); snprintf(sql, sizeof(sql), "%ld\t%d\t%d\t%d\t%s\t%s\t%s\n", agent_pk, *(int*)pair_second(current_file), copy_entry_start(entry), copy_entry_end(entry), buf, hash, copy_entry_type(entry)); fo_sqlCopyAdd(sqlcpy, sql); } else { fprintf(cout, "\t[%d:%d:%s] '%s'", copy_entry_start(entry), copy_entry_end(entry), copy_entry_type(entry), copy_entry_text(entry)); if(copy_entry_text(entry)[strlen(copy_entry_text(entry)) - 1] != '\n') { fprintf(cout, "\n"); } } } } fclose(input_fp); fo_scheduler_heart(1); }
/** * \brief Get the mimetype for a package * \param argc the number of command line arguments * \param argv the command line arguments * \return 0 on a successful program execution */ int main(int argc, char *argv[]) { int arg; char *Parm = NULL; char *Path = NULL; int c; char *agent_desc = "Determines mimetype for each file"; int pfile_count = 0; int Agent_pk; int ars_pk = 0; int upload_pk = 0; // the upload primary key int user_pk = 0; char *AgentARSName = "mimetype_ars"; int rv; PGresult *result; char sqlbuf[1024]; int CmdlineFlag = 0; ///< run from command line flag, 1 yes, 0 not char *COMMIT_HASH; char *VERSION; char agent_rev[MAXCMD]; /* initialize the scheduler connection */ fo_scheduler_connect(&argc, argv, &pgConn); /* Process command-line */ while((c = getopt(argc,argv,"iCc:hvV")) != -1) { switch(c) { case 'i': PQfinish(pgConn); return(0); case 'c': /* do nothing with this option */ break; case 'C': CmdlineFlag = 1; break; case 'v': agent_verbose++; break; case 'V': printf("%s", BuildVersion); PQfinish(pgConn); return(0); default: Usage(argv[0]); PQfinish(pgConn); exit(-1); } } COMMIT_HASH = fo_sysconfig("mimetype", "COMMIT_HASH"); VERSION = fo_sysconfig("mimetype", "VERSION"); sprintf(agent_rev, "%s.%s", VERSION, COMMIT_HASH); /* Get the Agent Key from the DB */ Agent_pk = fo_GetAgentKey(pgConn, basename(argv[0]), 0, agent_rev, agent_desc); FMimetype = fopen("/etc/mime.types","rb"); if (!FMimetype) { LOG_WARNING("Unable to open /etc/mime.types\n"); } MagicCookie = magic_open(MAGIC_PRESERVE_ATIME|MAGIC_MIME); if (MagicCookie == NULL) { LOG_FATAL("Failed to initialize magic cookie\n"); PQfinish(pgConn); exit(-1); } if (magic_load(MagicCookie,NULL) != 0) { LOG_FATAL("Failed to load magic file: UnMagic\n"); PQfinish(pgConn); exit(-1); } /* Run from the command-line (for testing) */ for(arg=optind; arg < argc; arg++) { Akey = -1; memset(A,'\0',sizeof(A)); strncpy(A,argv[arg],sizeof(A)); DBCheckMime(A); } /* Run from scheduler! */ if (0 == CmdlineFlag) { user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */ while(fo_scheduler_next()) { /* get piece of information, including upload_pk, others */ Parm = fo_scheduler_current(); if (Parm && Parm[0]) { upload_pk = atoi(Parm); /* Check Permissions */ if (GetUploadPerm(pgConn, upload_pk, user_pk) < PERM_WRITE) { LOG_ERROR("You have no update permissions on upload %d", upload_pk); continue; } /* does ars table exist? * If not, create it. */ rv = fo_tableExists(pgConn, AgentARSName); if (!rv) { rv = fo_CreateARSTable(pgConn, AgentARSName); if (!rv) return(0); } /* check ars table if this is duplicate request*/ memset(sqlbuf, 0, sizeof(sqlbuf)); snprintf(sqlbuf, sizeof(sqlbuf), "select ars_pk from mimetype_ars,agent \ where agent_pk=agent_fk and ars_success=true \ and upload_fk='%d' and agent_fk='%d'", upload_pk, Agent_pk); result = PQexec(pgConn, sqlbuf); if (fo_checkPQresult(pgConn, result, sqlbuf, __FILE__, __LINE__)) exit(-1); if (PQntuples(result) > 0) { PQclear(result); LOG_WARNING("Ignoring requested mimetype analysis of upload %d - Results are already in database.\n",upload_pk); continue; } PQclear(result); /* Record analysis start in mimetype_ars, the mimetype audit trail. */ ars_pk = fo_WriteARS(pgConn, ars_pk, upload_pk, Agent_pk, AgentARSName, 0, 0); /* get all pfile ids on a upload record */ memset(sqlbuf, 0, sizeof(sqlbuf)); snprintf(sqlbuf, sizeof(sqlbuf), "SELECT DISTINCT(pfile_pk) as Akey, pfile_sha1 || '.' || pfile_md5 || '.' || pfile_size AS A FROM uploadtree, pfile WHERE uploadtree.pfile_fk = pfile.pfile_pk AND pfile_mimetypefk is NULL AND upload_fk = '%d';", upload_pk); result = PQexec(pgConn, sqlbuf); if (fo_checkPQresult(pgConn, result, sqlbuf, __FILE__, __LINE__)) exit(-1); pfile_count = PQntuples(result); int i; for(i=0; i < pfile_count; i++) { Akey = atoi(PQgetvalue(result, i, 0)); strncpy(A, PQgetvalue(result, i, 1), sizeof(A)); if (Akey <= 0 || A[0]=='\0') { printf("ERROR: Data is in an unknown format.\n"); PQfinish(pgConn); exit(-1); } /* Process the repository file */ /* Find the path */ Path = fo_RepMkPath("files",A); if (Path && fo_RepExist("files",A)) { /* Get the mimetype! */ DBCheckMime(Path); } else { printf("ERROR pfile %d Unable to process.\n",Akey); printf("LOG pfile %d File '%s' not found.\n",Akey,A); PQfinish(pgConn); exit(-1); } /* Clean up Path memory */ if(Path) { free(Path); Path = NULL; } fo_scheduler_heart(1); } PQclear(result); /* Record analysis success in mimetype_ars. */ if (ars_pk) fo_WriteARS(pgConn, ars_pk, upload_pk, Agent_pk, AgentARSName, 0, 1); } } } /* if run from scheduler */ /* Clean up */ if (FMimetype) fclose(FMimetype); magic_close(MagicCookie); if (DBMime) PQclear(DBMime); if (pgConn) PQfinish(pgConn); /* after cleaning up agent, disconnect from the scheduler, this doesn't return */ fo_scheduler_disconnect(0); return(0); } /* main() */
int main (int argc, char *argv[]) { int arg; char *Parm = NULL; char *TempFileDir=NULL; int c; int InitFlag=0; int CmdlineFlag = 0; /** run from command line flag, 1 yes, 0 not */ int user_pk; char *agent_desc = "Network downloader. Uses wget(1)."; memset(GlobalTempFile,'\0',MAXCMD); memset(GlobalURL,'\0',MAXCMD); memset(GlobalParam,'\0',MAXCMD); memset(GlobalType,'\0',MAXCMD); GlobalUploadKey = -1; int upload_pk = 0; // the upload primary key //int Agent_pk; char *SVN_REV; char *VERSION; char agent_rev[MAXCMD]; /* open the connection to the scheduler and configuration */ fo_scheduler_connect(&argc, argv, &pgConn); /* Process command-line */ while((c = getopt(argc,argv,"d:Gg:ik:A:R:l:Cc:Vvh")) != -1) { switch(c) { case 'd': TempFileDir = PathCheck(optarg); break; case 'g': { struct group *SG; SG = getgrnam(optarg); if (SG) ForceGroup = SG->gr_gid; } break; case 'G': GlobalImportGold=0; break; case 'i': InitFlag=1; break; case 'k': GlobalUploadKey = atol(optarg); if (!GlobalTempFile[0]) strcpy(GlobalTempFile,"wget.default_download"); break; case 'A': sprintf(GlobalParam, "%s -A %s ",GlobalParam, optarg); break; case 'R': sprintf(GlobalParam, "%s -R %s ",GlobalParam, optarg); break; case 'l': sprintf(GlobalParam, "%s -l %s ",GlobalParam, optarg); break; case 'c': break; /* handled by fo_scheduler_connect() */ case 'C': CmdlineFlag = 1; break; case 'v': break; case 'V': printf("%s", BuildVersion); SafeExit(0); default: Usage(argv[0]); SafeExit(-1); } } if (argc - optind > 1) { Usage(argv[0]); SafeExit(-1); } /* When initializing the DB, don't do anything else */ if (InitFlag) { if (pgConn) PQfinish(pgConn); SafeExit(0); } SVN_REV = fo_sysconfig("wget_agent", "SVN_REV"); VERSION = fo_sysconfig("wget_agent", "VERSION"); sprintf(agent_rev, "%s.%s", VERSION, SVN_REV); /* Get the Agent Key from the DB */ fo_GetAgentKey(pgConn, basename(argv[0]), GlobalUploadKey, agent_rev, agent_desc); /** get proxy */ GetProxy(); /* Run from the command-line (for testing) */ for(arg=optind; arg < argc; arg++) { memset(GlobalURL,'\0',sizeof(GlobalURL)); strncpy(GlobalURL,argv[arg],sizeof(GlobalURL)); /* If the file contains "://" then assume it is a URL. Else, assume it is a file. */ LOG_VERBOSE0("Command-line: %s",GlobalURL); if (strstr(GlobalURL,"://")) { fo_scheduler_heart(1); LOG_VERBOSE0("It's a URL"); if (GetURL(GlobalTempFile,GlobalURL,TempFileDir) != 0) { LOG_FATAL("Download of %s failed.",GlobalURL); SafeExit(21); } if (GlobalUploadKey != -1) { DBLoadGold(); } unlink(GlobalTempFile); } else /* must be a file */ { LOG_VERBOSE0("It's a file -- GlobalUploadKey = %ld",GlobalUploadKey); if (GlobalUploadKey != -1) { memcpy(GlobalTempFile,GlobalURL,MAXCMD); DBLoadGold(); } } } /* Run from scheduler! */ if (0 == CmdlineFlag) { user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */ while(fo_scheduler_next()) { Parm = fo_scheduler_current(); /* get piece of information, including upload_pk, downloadfile url, and parameters */ if (Parm && Parm[0]) { fo_scheduler_heart(1); /* set globals: uploadpk, downloadfile url, parameters */ SetEnv(Parm,TempFileDir); upload_pk = GlobalUploadKey; /* Check Permissions */ if (GetUploadPerm(pgConn, upload_pk, user_pk) < PERM_WRITE) { LOG_ERROR("You have no update permissions on upload %d", upload_pk); continue; } char TempDir[MAXCMD]; memset(TempDir,'\0',MAXCMD); snprintf(TempDir, MAXCMD-1, "%s/wget", TempFileDir); // /var/local/lib/fossology/agents/wget struct stat Status; if (GlobalType[0]) { if (GetVersionControl() == 0) { DBLoadGold(); unlink(GlobalTempFile); } else { LOG_FATAL("upload %ld File retrieval failed: uploadpk=%ld tempfile=%s URL=%s Type=%s", GlobalUploadKey,GlobalUploadKey,GlobalTempFile,GlobalURL, GlobalType); SafeExit(23); } } else if (strstr(GlobalURL, "*") || stat(GlobalURL, &Status) == 0) { if (!Archivefs(GlobalURL, GlobalTempFile, TempFileDir, Status)) { LOG_FATAL("Failed to archieve. GlobalURL, GlobalTempFile, TempFileDir are: %s, %s, %s, " "Mode is: %lo (octal)\n", GlobalURL, GlobalTempFile, TempFileDir, (unsigned long) Status.st_mode); SafeExit(50); } DBLoadGold(); unlink(GlobalTempFile); } else { if (GetURL(GlobalTempFile,GlobalURL,TempDir) == 0) { DBLoadGold(); unlink(GlobalTempFile); } else { LOG_FATAL("upload %ld File retrieval failed: uploadpk=%ld tempfile=%s URL=%s", GlobalUploadKey,GlobalUploadKey,GlobalTempFile,GlobalURL); SafeExit(22); } } } } } /* if run from scheduler */ SafeExit(0); exit(0); /* to prevent compiler warning */ } /* main() */