コード例 #1
0
ファイル: util.c プロジェクト: fossology/fossology
/**
 * \brief process the jobs from scheduler
 *
 * -# Read the jobs from the scheduler using fo_scheduler_next().
 * -# Get the permission level of the current user.
 * -# Parse the parameters and process
 * \see fo_scheduler_next()
 * \see readAndProcessParameter()
 */
void doSchedulerTasks()
{
  char *Parm = NULL;
  char SQL[MAXSQL];
  PGresult *result;
  int userId = -1;
  int userPerm = -1;

  while(fo_scheduler_next())
  {
    Parm = fo_scheduler_current();
    userId = fo_scheduler_userID();

    /* get perm level of user */
    snprintf(SQL,MAXSQL,"SELECT user_perm FROM users WHERE user_pk='%d';", userId);
    result = PQexec(pgConn, SQL);
    if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__) || !PQntuples(result))
    {
      exitNow(0);
    }
    userPerm = atoi(PQgetvalue(result, 0, 0));
    PQclear(result);

    int returnCode = readAndProcessParameter(Parm, userId, userPerm);
    if (returnCode != 0)
    {
      /* Loglevel is to high, but scheduler expects FATAL log message before exit */
      LOG_FATAL("Due to permission problems, the delagent was not able to list or delete the requested objects or they did not exist.");
      exitNow(returnCode);
    }
  }
}
コード例 #2
0
ファイル: simple.c プロジェクト: 7hibault/fossology
int main(int argc, char** argv)
{
  fo_scheduler_connect(&argc, argv, NULL);
  if(fo_scheduler_next() == NULL)
    fo_scheduler_disconnect(0);
  else
    fo_scheduler_disconnect(-1);

  return 0;
}
コード例 #3
0
ファイル: no_check.c プロジェクト: fossology/fossology
int main(int argc, char** argv)
{
  fo_scheduler_connect(&argc, argv, NULL);

  while(1)
  {
    fo_scheduler_next();
  }

  fo_scheduler_disconnect(0);
  return 0;
}
コード例 #4
0
ファイル: regexscan.c プロジェクト: 7hibault/fossology
int	main	(int argc, char *argv[])
{
  int  nonoptargs;
  int  c, retCode;

  regex_t regex;

  char regexStr[1024];  /* string storage for the regex expression */
  bool regexSet = false;

  char fileName[1000];
  FILE *scanFilePtr;

  char uploadNum[10];

  int  scannedCount = 0;

  int   user_pk;
  long  UploadPK=-1;

  char *COMMIT_HASH;
  char *VERSION;
  char agent_rev[myBUFSIZ];

  /* connect to scheduler.  Noop if not run from scheduler.  */
  fo_scheduler_connect(&argc, argv, &pgConn);

/*
  Version reporting.
*/
  COMMIT_HASH = fo_sysconfig("regexscan", "COMMIT_HASH");
  VERSION = fo_sysconfig("regexscan", "VERSION");
  sprintf(agent_rev, "%s.%s", VERSION, COMMIT_HASH);
#ifdef REGEX_DEBUG
  fprintf(stdout, "regexscan reports version info as '%s.%s'.\n", VERSION, COMMIT_HASH);
#endif

  /* Process command-line */
  while((c = getopt(argc,argv,"chir:v")) != -1)
  {
    switch(c)
    {
    case 'c':
      break;  /* handled by fo_scheduler_connect()  */
    case 'i':
      PQfinish(pgConn);
      return(0);
    case 'r':
      sprintf(regexStr, "%s", optarg);
      regexSet = true;
      break;
    case 'v':
      agent_verbose++;
      break;
    case 'h':
    default:
      Usage(argv[0]);
      fflush(stdout);
      PQfinish(pgConn);
      exit(-1);
    }
  }

  /* Sanity check for regex value required here. */
  if (!regexSet)
  {
    fprintf (stderr, "No regex value has been requested!\n");
    PQfinish(pgConn);
    fo_scheduler_disconnect(0);
    return 1;
  }

  /* process filename after switches. How many non-option arguments are there ? */
  nonoptargs = argc - optind; /* total argument count minus the option count */

  if (nonoptargs == 0)
  {
    /* Assume it was a scheduler call */
    user_pk = fo_scheduler_userID();

    while(fo_scheduler_next())
    {
      UploadPK = atol(fo_scheduler_current());

      printf("UploadPK is: %ld\n", UploadPK);
      sprintf(uploadNum, "%ld", UploadPK);
	    scannedCount = regexScanUpload(uploadNum, regexStr);
      if (scannedCount == 0)
      {
        fprintf(stderr, "Failed to successfully scan: upload - %s!\n", uploadNum);
      }
    }
  }
  else
  {
    /* File access initialization - For Stage 3 use first arg as fileName */
    sprintf(fileName, "%s", argv[optind]);      /* Grab first non-switch argument as filename */

    scanFilePtr = fopen(fileName, "r");
    if (!scanFilePtr)
    {
      fprintf(stderr, "ERROR: Unable to open '%s'\n", fileName);
      PQfinish(pgConn);
      fo_scheduler_disconnect(0);
    }

    /* Compile the regex for improved performance */
    retCode = regcomp(&regex, regexStr, REG_ICASE+REG_EXTENDED);
    if (retCode)
    {
      fprintf(stderr, "regex %s failed to compile\n", regexStr);
      PQfinish(pgConn);
      fo_scheduler_disconnect(0);
    }

    /* Now call the function that scans a file for a regex */
    retCode = regexScan(&regex, (char *)regexStr, scanFilePtr, (char *)fileName);
//    retCode = regexScan(uploadNum, regexStr);
    if (retCode != 0)
    {
      fprintf(stderr, "Failed to successfully scan: %s!\n", fileName);
    }

  }

  PQfinish(pgConn);
  fo_scheduler_disconnect(0);

  return 0;
} /* main() */
コード例 #5
0
ファイル: buckets.c プロジェクト: fossology/fossology
int main(int argc, char **argv)
{
  char *agentDesc = "Bucket agent";
  int cmdopt;
  int verbose = 0;
  int ReadFromStdin = 1;
  int head_uploadtree_pk = 0;
  PGconn *pgConn;
  PGresult *topresult;
  PGresult *result;
  char sqlbuf[512];
  char *Delims = ",= \t\n\r";
  char *token, *saveptr;
  int agent_pk = 0;
  int nomos_agent_pk = 0;
  int bucketpool_pk = 0;
  int ars_pk = 0;
  int readnum = 0;
  int rv;
  int hasPrules;
  int user_pk = 0;
  char *bucketpool_name;
  char *COMMIT_HASH;
  char *VERSION;
  char *uploadtree_tablename;
  char agent_rev[myBUFSIZ];
  int rerun = 0;


//  int *bucketList;
  pbucketdef_t bucketDefArray = 0;
  pbucketdef_t tmpbucketDefArray = 0;
  cacheroot_t  cacheroot;
  uploadtree_t  uploadtree;
  uploadtree.upload_fk = 0;

  /* connect to the scheduler */
  fo_scheduler_connect(&argc, argv, &pgConn);
  user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */

  /* command line options */
  while ((cmdopt = getopt(argc, argv, "rin:p:t:u:vc:hV")) != -1)
  {
    switch (cmdopt)
    {
      case 'i': /* "Initialize" */
            PQfinish(pgConn);
            exit(0);
      case 'n': /* bucketpool_name  */
            ReadFromStdin = 0;
            bucketpool_name = optarg;
            /* find the highest rev active bucketpool_pk */
            if (!bucketpool_pk)
            {
              bucketpool_pk = getBucketpool_pk(pgConn, bucketpool_name);
              if (!bucketpool_pk)
                printf("%s is not an active bucketpool name.\n", bucketpool_name);
            }
            break;
      case 'p': /* bucketpool_pk */
            ReadFromStdin = 0;
            bucketpool_pk = atoi(optarg);
            /* validate bucketpool_pk */
            sprintf(sqlbuf, "select bucketpool_pk from bucketpool where bucketpool_pk=%d and active='Y'", bucketpool_pk);
            bucketpool_pk = validate_pk(pgConn, sqlbuf);
            if (!bucketpool_pk)
              printf("%d is not an active bucketpool_pk.\n", atoi(optarg));
            break;
      case 't': /* uploadtree_pk */
            ReadFromStdin = 0;
            if (uploadtree.upload_fk) break;
            head_uploadtree_pk = atoi(optarg);
            /* validate bucketpool_pk */
            sprintf(sqlbuf, "select uploadtree_pk from uploadtree where uploadtree_pk=%d", head_uploadtree_pk);
            head_uploadtree_pk = validate_pk(pgConn, sqlbuf);
            if (!head_uploadtree_pk)
              printf("%d is not an active uploadtree_pk.\n", atoi(optarg));
            break;
      case 'u': /* upload_pk */
            ReadFromStdin = 0;
            if (!head_uploadtree_pk)
            {
              uploadtree.upload_fk = atoi(optarg);
              /* validate upload_pk  and get uploadtree_pk  */
              sprintf(sqlbuf, "select upload_pk from upload where upload_pk=%d", uploadtree.upload_fk);
              uploadtree.upload_fk = validate_pk(pgConn, sqlbuf);
              if (!uploadtree.upload_fk)
                printf("%d is not an valid upload_pk.\n", atoi(optarg));
              else
              {
                sprintf(sqlbuf, "select uploadtree_pk from uploadtree where upload_fk=%d and parent is null", uploadtree.upload_fk);
                head_uploadtree_pk = validate_pk(pgConn, sqlbuf);
              }
            }
            break;
      case 'v': /* verbose output for debugging  */
            verbose++;
            break;
      case 'c': break; /* handled by fo_scheduler_connect() */
      case 'r':
            rerun = 1; /** rerun bucket */
            break;
      case 'V': /* print version info */
            printf("%s", BuildVersion);
            PQfinish(pgConn);
            exit(0);
      default:
            Usage(argv[0]);
            PQfinish(pgConn);
            exit(-1);
    }
  }
  debug = verbose;

  /*** validate command line ***/
  if (!bucketpool_pk && !ReadFromStdin)
  {
    printf("FATAL: You must specify an active bucketpool.\n");
    Usage(argv[0]);
    exit(-1);
  }
  if (!head_uploadtree_pk && !ReadFromStdin)
  {
    printf("FATAL: You must specify a valid uploadtree_pk or upload_pk.\n");
    Usage(argv[0]);
    exit(-1);
  }

  /* get agent pk
   * Note, if GetAgentKey fails, this process will exit.
   */
  COMMIT_HASH = fo_sysconfig("buckets", "COMMIT_HASH");
  VERSION = fo_sysconfig("buckets", "VERSION");
  sprintf(agent_rev, "%s.%s", VERSION, COMMIT_HASH);
  agent_pk = fo_GetAgentKey(pgConn, basename(argv[0]), uploadtree.upload_fk, agent_rev, agentDesc);

  /*** Initialize the license_ref table cache ***/
  /* Build the license ref cache to hold 2**11 (2048) licenses.
     This MUST be a power of 2.
   */
  cacheroot.maxnodes = 2<<11;
  cacheroot.nodes = calloc(cacheroot.maxnodes, sizeof(cachenode_t));
  if (!lrcache_init(pgConn, &cacheroot))
  {
    printf("FATAL: Bucket agent could not allocate license_ref table cache.\n");
    exit(1);
  }


  /* main processing loop */
  while(++readnum)
  {
    uploadtree.upload_fk = 0;
    if (ReadFromStdin)
    {
      bucketpool_pk = 0;

      /* Read the bucketpool_pk and upload_pk from stdin.
       * Format looks like 'bppk=123, upk=987'
       */
      if (!fo_scheduler_next()) break;

      token = strtok_r(fo_scheduler_current(), Delims, &saveptr);
      while (token && (!uploadtree.upload_fk || !bucketpool_pk))
      {
        if (strcmp(token, "bppk") == 0)
        {
          bucketpool_pk = atoi(strtok_r(NULL, Delims, &saveptr));
        }
        else
        if (strcmp(token, "upk") == 0)
        {
          uploadtree.upload_fk = atoi(strtok_r(NULL, Delims, &saveptr));
        }
        token = strtok_r(NULL, Delims, &saveptr);
      }

      /* Check Permissions */
      if (GetUploadPerm(pgConn, uploadtree.upload_fk, user_pk) < PERM_WRITE)
      {
        LOG_ERROR("You have no update permissions on upload %d", uploadtree.upload_fk);
        continue;
      }

      /* From the upload_pk, get the head of the uploadtree, pfile_pk and ufile_name  */
      sprintf(sqlbuf, "select uploadtree_pk, pfile_fk, ufile_name, ufile_mode,lft,rgt from uploadtree \
             where upload_fk='%d' and parent is null limit 1", uploadtree.upload_fk);
      topresult = PQexec(pgConn, sqlbuf);
      if (fo_checkPQresult(pgConn, topresult, sqlbuf, agentDesc, __LINE__)) return -1;
      if (PQntuples(topresult) == 0)
      {
        printf("ERROR: %s.%s missing upload_pk %d.\nsql: %s",
               __FILE__, agentDesc, uploadtree.upload_fk, sqlbuf);
        PQclear(topresult);
        continue;
      }
      head_uploadtree_pk = atol(PQgetvalue(topresult, 0, 0));
      uploadtree.uploadtree_pk = head_uploadtree_pk;
      uploadtree.upload_fk = uploadtree.upload_fk;
      uploadtree.pfile_fk = atol(PQgetvalue(topresult, 0, 1));
      uploadtree.ufile_name = strdup(PQgetvalue(topresult, 0, 2));
      uploadtree.ufile_mode = atoi(PQgetvalue(topresult, 0, 3));
      uploadtree.lft = atoi(PQgetvalue(topresult, 0, 4));
      uploadtree.rgt = atoi(PQgetvalue(topresult, 0, 5));
      PQclear(topresult);
    } /* end ReadFromStdin */
    else
    {
      /* Only one input to process if from command line, so terminate if it's been done */
      if (readnum > 1) break;

      /* not reading from stdin
       * Get the pfile, and ufile_name for head_uploadtree_pk
       */
      sprintf(sqlbuf, "select pfile_fk, ufile_name, ufile_mode,lft,rgt, upload_fk from uploadtree where uploadtree_pk=%d", head_uploadtree_pk);
      topresult = PQexec(pgConn, sqlbuf);
      if (fo_checkPQresult(pgConn, topresult, sqlbuf, agentDesc, __LINE__))
      {
        free(uploadtree.ufile_name);
        return -1;
      }
      if (PQntuples(topresult) == 0)
      {
        printf("FATAL: %s.%s missing root uploadtree_pk %d\n",
               __FILE__, agentDesc, head_uploadtree_pk);
        PQclear(topresult);
        continue;
      }
      uploadtree.uploadtree_pk = head_uploadtree_pk;
      uploadtree.pfile_fk = atol(PQgetvalue(topresult, 0, 0));
      uploadtree.ufile_name = strdup(PQgetvalue(topresult, 0, 1));
      uploadtree.ufile_mode = atoi(PQgetvalue(topresult, 0, 2));
      uploadtree.lft = atoi(PQgetvalue(topresult, 0, 3));
      uploadtree.rgt = atoi(PQgetvalue(topresult, 0, 4));
      uploadtree.upload_fk = atoi(PQgetvalue(topresult, 0, 5));
      PQclear(topresult);
    }

    /* Find the most recent nomos data for this upload.  That's what we want to use
         to process the buckets.
     */
    nomos_agent_pk = LatestNomosAgent(pgConn, uploadtree.upload_fk);
    if (nomos_agent_pk == 0)
    {
      printf("WARNING: Bucket agent called on treeitem (%d), but the latest nomos agent hasn't created any license data for this tree.\n",
            head_uploadtree_pk);
      continue;
    }

    /* at this point we know:
     * bucketpool_pk, bucket agent_pk, nomos agent_pk, upload_pk,
     * pfile_pk, and head_uploadtree_pk (the uploadtree_pk of the head tree to scan)
     */

    /* Has the upload already been processed?  If so, we are done.
       Don't even bother to create a bucket_ars entry.
     */
    switch (UploadProcessed(pgConn, agent_pk, nomos_agent_pk, uploadtree.pfile_fk, head_uploadtree_pk, uploadtree.upload_fk, bucketpool_pk))
    {
      case 1:  /* upload has already been processed */
        if (1 == rerun) break;
        printf("LOG: Duplicate request for bucket agent to process upload_pk: %d, uploadtree_pk: %d, bucketpool_pk: %d, bucket agent_pk: %d, nomos agent_pk: %d, pfile_pk: %d ignored.\n",
             uploadtree.upload_fk, head_uploadtree_pk, bucketpool_pk, agent_pk, nomos_agent_pk, uploadtree.pfile_fk);
        continue;
      case -1: /* SQL error, UploadProcessed() wrote error message */
        continue;
      case 0:  /* upload has not been processed */
        break;
    }

    /*** Initialize the Bucket Definition List bucketDefArray  ***/
    bucketDefArray = initBuckets(pgConn, bucketpool_pk, &cacheroot);
    if (bucketDefArray == 0)
    {
      printf("FATAL: %s.%d Bucket definition for pool %d could not be initialized.\n",
             __FILE__, __LINE__, bucketpool_pk);
      exit(-2);
    }
    bucketDefArray->nomos_agent_pk = nomos_agent_pk;
    bucketDefArray->bucket_agent_pk = agent_pk;

    /* Find the correct uploadtree table name */
    uploadtree_tablename = GetUploadtreeTableName(pgConn, uploadtree.upload_fk);
    if (!(uploadtree_tablename))
    {
      LOG_FATAL("buckets passed invalid upload, upload_pk = %d", uploadtree.upload_fk);
      return(-110);
    }

    /* set uploadtree_tablename in all the bucket definition structs */
    for (tmpbucketDefArray = bucketDefArray; tmpbucketDefArray->bucket_pk; tmpbucketDefArray++)
    {
      tmpbucketDefArray->uploadtree_tablename = uploadtree_tablename;
    }

    /* loop through rules (bucket defs) to see if there are any package only rules */
    hasPrules = 0;
    for (tmpbucketDefArray = bucketDefArray; tmpbucketDefArray->bucket_pk; tmpbucketDefArray++)
      if (tmpbucketDefArray->applies_to == 'p')
      {
        hasPrules = 1;
        break;
      }

    /*** END initializing bucketDefArray  ***/

    /*** Initialize DEB_SOURCE and DEB_BINARY  ***/
    sprintf(sqlbuf, "select mimetype_pk from mimetype where mimetype_name='application/x-debian-package'");
    result = PQexec(pgConn, sqlbuf);
    if (fo_checkPQresult(pgConn, result, sqlbuf, __FILE__, __LINE__)) return -1;
    if (PQntuples(result) == 0)
    {
      printf("FATAL: (%s.%d) Missing application/x-debian-package mimetype.\n",__FILE__,__LINE__);
      return -1;
    }
    DEB_BINARY = atoi(PQgetvalue(result, 0, 0));
    PQclear(result);

    sprintf(sqlbuf, "select mimetype_pk from mimetype where mimetype_name='application/x-debian-source'");
    result = PQexec(pgConn, sqlbuf);
    if (fo_checkPQresult(pgConn, result, sqlbuf, __FILE__, __LINE__)) return -1;
    if (PQntuples(result) == 0)
    {
      printf("FATAL: (%s.%d) Missing application/x-debian-source mimetype.\n",__FILE__,__LINE__);
      return -1;
    }
    DEB_SOURCE = atoi(PQgetvalue(result, 0, 0));
    PQclear(result);
    /*** END Initialize DEB_SOURCE and DEB_BINARY  ***/

    /*** Record analysis start in bucket_ars, the bucket audit trail. ***/
    if (0 == rerun) { // do not have any bucket scan on this upload
      snprintf(sqlbuf, sizeof(sqlbuf),
          "insert into bucket_ars (agent_fk, upload_fk, ars_success, nomosagent_fk, bucketpool_fk) values(%d,%d,'%s',%d,%d)",
          agent_pk, uploadtree.upload_fk, "false", nomos_agent_pk, bucketpool_pk);
      if (debug)
        printf("%s(%d): %s\n", __FILE__, __LINE__, sqlbuf);

      result = PQexec(pgConn, sqlbuf);
      if (fo_checkPQcommand(pgConn, result, sqlbuf, __FILE__ ,__LINE__)) return -1;
      PQclear(result);

      /* retrieve the ars_pk of the newly inserted record */
      sprintf(sqlbuf, "select ars_pk from bucket_ars where agent_fk='%d' and upload_fk='%d' and ars_success='%s' and nomosagent_fk='%d' \
          and bucketpool_fk='%d' and ars_endtime is null \
          order by ars_starttime desc limit 1",
          agent_pk, uploadtree.upload_fk, "false", nomos_agent_pk, bucketpool_pk);
      result = PQexec(pgConn, sqlbuf);
      if (fo_checkPQresult(pgConn, result, sqlbuf, __FILE__, __LINE__)) return -1;
      if (PQntuples(result) == 0)
      {
        printf("FATAL: (%s.%d) Missing bucket_ars record.\n%s\n",__FILE__,__LINE__,sqlbuf);
        return -1;
      }
      ars_pk = atol(PQgetvalue(result, 0, 0));
      PQclear(result);
    }
    /*** END bucket_ars insert  ***/

    if (debug) printf("%s sql: %s\n",__FILE__, sqlbuf);

    /* process the tree for buckets
       Do this as a single transaction, therefore this agent must be
       run as a single thread.  This will prevent the scheduler from
       consuming excess time (this is a fast agent), and allow this
       process to update bucket_ars.
     */
    rv = walkTree(pgConn, bucketDefArray, agent_pk, head_uploadtree_pk, 0,
             hasPrules);
    /* if no errors and top level is a container, process the container */
    if ((!rv) && (IsContainer(uploadtree.ufile_mode)))
    {
      rv = processFile(pgConn, bucketDefArray, &uploadtree, agent_pk, hasPrules);
    }

    /* Record analysis end in bucket_ars, the bucket audit trail. */
    if (0 == rerun && ars_pk)
    {
      if (rv)
        snprintf(sqlbuf, sizeof(sqlbuf),
                "update bucket_ars set ars_endtime=now(), ars_success=false where ars_pk='%d'",
                ars_pk);
      else
        snprintf(sqlbuf, sizeof(sqlbuf),
                "update bucket_ars set ars_endtime=now(), ars_success=true where ars_pk='%d'",
                ars_pk);

      if (debug)
        printf("%s(%d): %s\n", __FILE__, __LINE__, sqlbuf);

      result = PQexec(pgConn, sqlbuf);
      if (fo_checkPQcommand(pgConn, result, sqlbuf, __FILE__ ,__LINE__)) return -1;
      PQclear(result);
    }
  }  /* end of main processing loop */
コード例 #6
0
ファイル: mimetype.c プロジェクト: fossology/fossology
/**
 * \brief Get the mimetype for a package
 * \param argc the number of command line arguments
 * \param argv the command line arguments
 * \return 0 on a successful program execution
 */
int main(int argc, char *argv[])
{
  int arg;
  char *Parm = NULL;
  char *Path = NULL;
  int c;
  char *agent_desc = "Determines mimetype for each file";
  int pfile_count = 0;
  int Agent_pk;
  int ars_pk = 0;

  int upload_pk = 0;           // the upload primary key
  int user_pk = 0;
  char *AgentARSName = "mimetype_ars";
  int rv;
  PGresult *result;
  char sqlbuf[1024];
  int CmdlineFlag = 0;        ///< run from command line flag, 1 yes, 0 not
  char *COMMIT_HASH;
  char *VERSION;
  char agent_rev[MAXCMD];

  /* initialize the scheduler connection */
  fo_scheduler_connect(&argc, argv, &pgConn);

  /* Process command-line */
  while((c = getopt(argc,argv,"iCc:hvV")) != -1)
  {
    switch(c)
    {
      case 'i':
        PQfinish(pgConn);
        return(0);
      case 'c':
        /* do nothing with this option */
        break;
      case 'C':
        CmdlineFlag = 1;
        break;
      case 'v':
        agent_verbose++;
        break;
      case 'V':
        printf("%s", BuildVersion);
        PQfinish(pgConn);
        return(0);
      default:
        Usage(argv[0]);
        PQfinish(pgConn);
        exit(-1);
    }
  }

  COMMIT_HASH = fo_sysconfig("mimetype", "COMMIT_HASH");
  VERSION = fo_sysconfig("mimetype", "VERSION");
  sprintf(agent_rev, "%s.%s", VERSION, COMMIT_HASH);
  /* Get the Agent Key from the DB */
  Agent_pk = fo_GetAgentKey(pgConn, basename(argv[0]), 0, agent_rev, agent_desc);

  FMimetype = fopen("/etc/mime.types","rb");
  if (!FMimetype)
  {
    LOG_WARNING("Unable to open /etc/mime.types\n");
  }

  MagicCookie = magic_open(MAGIC_PRESERVE_ATIME|MAGIC_MIME);
  if (MagicCookie == NULL)
  {
    LOG_FATAL("Failed to initialize magic cookie\n");
    PQfinish(pgConn);
    exit(-1);
  }
  if (magic_load(MagicCookie,NULL) != 0)
  {
    LOG_FATAL("Failed to load magic file: UnMagic\n");
    PQfinish(pgConn);
    exit(-1);
  }

  /* Run from the command-line (for testing) */
  for(arg=optind; arg < argc; arg++)
  {
    Akey = -1;
    memset(A,'\0',sizeof(A));
    strncpy(A,argv[arg],sizeof(A));
    DBCheckMime(A);
  }

  /* Run from scheduler! */
  if (0 == CmdlineFlag)
  {
    user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */

    while(fo_scheduler_next())
    {
      /* get piece of information, including upload_pk, others */
      Parm = fo_scheduler_current();
      if (Parm && Parm[0])
      {
        upload_pk = atoi(Parm);

        /* Check Permissions */
        if (GetUploadPerm(pgConn, upload_pk, user_pk) < PERM_WRITE)
        {
          LOG_ERROR("You have no update permissions on upload %d", upload_pk);
          continue;
        }

        /* does ars table exist?
         * If not, create it.
         */
        rv = fo_tableExists(pgConn, AgentARSName);
        if (!rv)
        {
          rv = fo_CreateARSTable(pgConn, AgentARSName);
          if (!rv) return(0);
        }

        /* check ars table if this is duplicate request*/
        memset(sqlbuf, 0, sizeof(sqlbuf));
        snprintf(sqlbuf, sizeof(sqlbuf),
            "select ars_pk from mimetype_ars,agent \
            where agent_pk=agent_fk and ars_success=true \
            and upload_fk='%d' and agent_fk='%d'",
            upload_pk, Agent_pk);
        result = PQexec(pgConn, sqlbuf);
        if (fo_checkPQresult(pgConn, result, sqlbuf, __FILE__, __LINE__)) exit(-1);
        if (PQntuples(result) > 0)
        {
          PQclear(result);
          LOG_WARNING("Ignoring requested mimetype analysis of upload %d - Results are already in database.\n",upload_pk);
          continue;
        }
        PQclear(result);

        /* Record analysis start in mimetype_ars, the mimetype audit trail. */
        ars_pk = fo_WriteARS(pgConn, ars_pk, upload_pk, Agent_pk, AgentARSName, 0, 0);

        /* get all pfile ids on a upload record */
        memset(sqlbuf, 0, sizeof(sqlbuf));
        snprintf(sqlbuf, sizeof(sqlbuf), "SELECT DISTINCT(pfile_pk) as Akey, pfile_sha1 || '.' || pfile_md5 || '.' || pfile_size AS A FROM uploadtree, pfile WHERE uploadtree.pfile_fk = pfile.pfile_pk AND pfile_mimetypefk is NULL AND upload_fk = '%d';", upload_pk);
        result = PQexec(pgConn, sqlbuf);
        if (fo_checkPQresult(pgConn, result, sqlbuf, __FILE__, __LINE__)) exit(-1);
        pfile_count = PQntuples(result);
        int i;
        for(i=0; i < pfile_count; i++)
        {
          Akey = atoi(PQgetvalue(result, i, 0));
          strncpy(A, PQgetvalue(result, i, 1), sizeof(A));
          if (Akey <= 0 || A[0]=='\0')
          {
            printf("ERROR: Data is in an unknown format.\n");
            PQfinish(pgConn);
            exit(-1);
          }

          /* Process the repository file */
          /* Find the path */
          Path = fo_RepMkPath("files",A);
          if (Path && fo_RepExist("files",A))
          {
            /* Get the mimetype! */
            DBCheckMime(Path);
          }
          else
          {
            printf("ERROR pfile %d Unable to process.\n",Akey);
            printf("LOG pfile %d File '%s' not found.\n",Akey,A);
            PQfinish(pgConn);
            exit(-1);
          }
          /* Clean up Path memory */
          if(Path)
          {
            free(Path);
            Path = NULL;
          }
          fo_scheduler_heart(1);
        }
        PQclear(result);

        /* Record analysis success in mimetype_ars. */
        if (ars_pk) fo_WriteARS(pgConn, ars_pk, upload_pk, Agent_pk, AgentARSName, 0, 1);
      }
    }
  } /* if run from scheduler */

  /* Clean up */
  if (FMimetype) fclose(FMimetype);
  magic_close(MagicCookie);
  if (DBMime) PQclear(DBMime);
  if (pgConn) PQfinish(pgConn);
  /* after cleaning up agent, disconnect from the scheduler, this doesn't return */
  fo_scheduler_disconnect(0);
  return(0);
} /* main() */
コード例 #7
0
ファイル: main.c プロジェクト: rlintu/fossology
/**
 * \brief main function for the pkgagent
 *
 * There are 2 ways to use the pkgagent agent:
 *   1. Command Line Analysis :: test a rpm file from the command line
 *   2. Agent Based Analysis  :: run from the scheduler
 *
 * +-----------------------+
 * | Command Line Analysis |
 * +-----------------------+
 *
 * To analyze a rpm file from the command line:
 *   file :: if files are rpm package listed, display their meta data
 *   -v   :: verbose (-vv = more verbose)
 *
 *   example:
 *     $ ./pkgagent rpmfile
 *
 * +----------------------+
 * | Agent Based Analysis |
 * +----------------------+
 *
 * To run the pkgagent as an agent simply run with no command line args
 *   no file :: process data from the scheduler
 *   -i      :: initialize the database, then exit
 *
 *   example:
 *     $ upload_pk | ./pkgagent
 *
 * \param argc the number of command line arguments
 * \param argv the command line arguments
 * \return 0 on a successful program execution
 */
int	main	(int argc, char *argv[])
{
    int c;
    char *agent_desc = "Pulls metadata out of RPM or DEBIAN packages";
    //struct rpmpkginfo *glb_rpmpi;
    //struct debpkginfo *glb_debpi;
    int Agent_pk;
    int ars_pk = 0;

    int upload_pk = 0;           // the upload primary key
    int user_pk = 0;           // the upload primary key
    char *AgentARSName = "pkgagent_ars";
    int rv;
    PGresult *ars_result;
    char sqlbuf[1024];
    char *COMMIT_HASH;
    char *VERSION;
    char agent_rev[MAXCMD];
    int CmdlineFlag = 0; /* run from command line flag, 1 yes, 0 not */

    fo_scheduler_connect(&argc, argv, &db_conn);

    //glb_rpmpi = (struct rpmpkginfo *)malloc(sizeof(struct rpmpkginfo));
    //glb_debpi = (struct debpkginfo *)malloc(sizeof(struct debpkginfo));

    COMMIT_HASH = fo_sysconfig("pkgagent", "COMMIT_HASH");
    VERSION = fo_sysconfig("pkgagent", "VERSION");
    sprintf(agent_rev, "%s.%s", VERSION, COMMIT_HASH);
    Agent_pk = fo_GetAgentKey(db_conn, basename(argv[0]), 0, agent_rev, agent_desc);

    /* Process command-line */
    while((c = getopt(argc,argv,"ic:CvVh")) != -1)
    {
        switch(c)
        {
        case 'i':
            PQfinish(db_conn);  /* DB was opened above, now close it and exit */
            exit(0);
        case 'v':
            Verbose++;
            break;
        case 'c':
            break; /* handled by fo_scheduler_connect() */
        case 'C':
            CmdlineFlag = 1;
            break;
        case 'V':
            printf("%s", BuildVersion);
            PQfinish(db_conn);
            return(0);
        default:
            Usage(argv[0]);
            PQfinish(db_conn);
            exit(-1);
        }
    }
    /* If no args, run from scheduler! */
    if (CmdlineFlag == 0)
    {
        user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */

        while(fo_scheduler_next())
        {
            upload_pk = atoi(fo_scheduler_current());

            /* Check Permissions */
            if (GetUploadPerm(db_conn, upload_pk, user_pk) < PERM_WRITE)
            {
                LOG_ERROR("You have no update permissions on upload %d", upload_pk);
                continue;
            }

            if (Verbose) {
                printf("PKG: pkgagent read %d\n", upload_pk);
            }
            if (upload_pk ==0) continue;

            /* check if pkgagent ars table exist?
             * if exist, check duplicate request
             * if not exist, don't check duplicate request
             */
            rv = fo_tableExists(db_conn, AgentARSName);
            if (rv)
            {
                /* check ars table to see if this is duplicate request*/
                snprintf(sqlbuf, sizeof(sqlbuf),
                         "select ars_pk from pkgagent_ars,agent \
          where agent_pk=agent_fk and ars_success=true \
          and upload_fk='%d' and agent_fk='%d'",
                         upload_pk, Agent_pk);
                ars_result = PQexec(db_conn, sqlbuf);
                if (fo_checkPQresult(db_conn, ars_result, sqlbuf, __FILE__, __LINE__)) exit(-1);
                if (PQntuples(ars_result) > 0)
                {
                    PQclear(ars_result);
                    LOG_WARNING("Ignoring requested pkgagent analysis of upload %d - Results are already in database.\n",upload_pk);
                    continue;
                }
                PQclear(ars_result);
            }
            /* Record analysis start in pkgagent_ars, the pkgagent audit trail. */
            ars_pk = fo_WriteARS(db_conn, ars_pk, upload_pk, Agent_pk, AgentARSName, 0, 0);

            /* process the upload_pk pkgagent */
            if(ProcessUpload(upload_pk) != 0) return -1;

            /* Record analysis success in pkgagent_ars. */
            if (ars_pk) fo_WriteARS(db_conn, ars_pk, upload_pk, Agent_pk, AgentARSName, 0, 1);
        }
    }
    else
    {
        if (Verbose) {
            printf("DEBUG: running in cli mode, processing file(s)\n");
        }
        for (; optind < argc; optind++)
        {
            struct rpmpkginfo *rpmpi;
            rpmpi = (struct rpmpkginfo *)malloc(sizeof(struct rpmpkginfo));
            rpmReadConfigFiles(NULL, NULL);
            //if(ProcessUpload(atoi(argv[optind])) == 0)
            if(GetMetadata(argv[optind],rpmpi) != -1)
                printf("OK\n");
            else
                printf("Fail\n");
#ifdef _RPM_4_4_COMPAT
            rpmFreeCrypto();
            int i;
            for(i=0; i< rpmpi->req_size; i++)
                free(rpmpi->requires[i]);
#endif /* After RPM4.4 version*/
            free(rpmpi->requires);
            free(rpmpi);
            rpmFreeMacros(NULL);
        }
    }

    PQfinish(db_conn);
    fo_scheduler_disconnect(0);
    return(0);
} /* main() */
コード例 #8
0
ファイル: ununpack.c プロジェクト: 7hibault/fossology
int	main(int argc, char *argv[])
{
  int Pid;
  int c;
  int rvExist1=0, rvExist2=0;
  PGresult *result;
  char *NewDir=".";
  char *AgentName = "ununpack";
  char *AgentARSName = "ununpack_ars";
  char *agent_desc = "Unpacks archives (iso, tar, etc)";
  int   Recurse=0;
  int   ars_pk = 0;
  int   user_pk = 0;
  long  Pfile_size = 0;
  char *ListOutName=NULL;
  char *Fname = NULL;
  char *FnameCheck = NULL;
  char *COMMIT_HASH;
  char *VERSION;
  char agent_rev[PATH_MAX];
  struct stat Stat;

  /* connect to the scheduler */
  fo_scheduler_connect(&argc, argv, &pgConn);

  while((c = getopt(argc,argv,"ACc:d:FfHhL:m:PQiqRr:T:t:U:VvXx")) != -1)
  {
    switch(c)
    {
      case 'A':	SetContainerArtifact=0; break;
      case 'C':	ForceContinue=1; break;
      case 'c':	break;  /* handled by fo_scheduler_connect() */
      case 'd':	
        /* if there is a %U in the path, substitute a unique ID */
        NewDir=PathCheck(optarg);
        break;
      case 'F':	UseRepository=1; break;
      case 'f':	ForceDuplicate=1; break;
      case 'L':	ListOutName=optarg; break;
      case 'm':
        MaxThread = atoi(optarg);
        if (MaxThread < 1) MaxThread=1;
        break;
      case 'P':	PruneFiles=1; break;
      case 'R':	Recurse=-1; break;
      case 'r':	Recurse=atoi(optarg); break;
      case 'i':
        if (!IsExe("dpkg-source",Quiet))
          LOG_WARNING("dpkg-source is not available on this system.  This means that debian source packages will NOT be unpacked.");
        SafeExit(0);
        break; /* never reached */
      case 'Q':
        UseRepository=1;

        user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */

        /* Get the upload_pk from the scheduler */
        if((Upload_Pk = fo_scheduler_next()) == NULL) SafeExit(0);
        break;
      case 'q':	Quiet=1; break;
      case 'T':
        memset(REP_GOLD,0,sizeof(REP_GOLD));
        strncpy(REP_GOLD,optarg,sizeof(REP_GOLD)-1);
        break;
      case 't':
        memset(REP_FILES,0,sizeof(REP_FILES));
        strncpy(REP_FILES,optarg,sizeof(REP_FILES)-1);
        break;
      case 'U':	
        UseRepository = 1;
        Recurse = -1;
        Upload_Pk = optarg; 
        break;
      case 'V': printf("%s", BuildVersion);SafeExit(0);
      case 'v':	Verbose++; break;
      case 'X':	UnlinkSource=1; break;
      case 'x':	UnlinkAll=1; break;
      default:
        Usage(argv[0], BuildVersion);
        SafeExit(25);
    }
  }

  /* Open DB and Initialize CMD table */
  if (UseRepository) 
  {
    /* Check Permissions */
    if (GetUploadPerm(pgConn, atoi(Upload_Pk), user_pk) < PERM_WRITE)
    {
      LOG_ERROR("You have no update permissions on upload %s", Upload_Pk);
      SafeExit(100);
    }
        
    COMMIT_HASH = fo_sysconfig(AgentName, "COMMIT_HASH");
    VERSION = fo_sysconfig(AgentName, "VERSION");
    sprintf(agent_rev, "%s.%s", VERSION, COMMIT_HASH);
    /* Get the unpack agent key */
    agent_pk = fo_GetAgentKey(pgConn, AgentName, atoi(Upload_Pk), agent_rev,agent_desc);

    InitCmd();

    /* Make sure ars table exists */
    if (!fo_CreateARSTable(pgConn, AgentARSName)) SafeExit(0);

    /* Has this user previously unpacked this upload_pk successfully?
     *    In this case we are done.  No new ars record is needed since no
     *    processing is initiated.
     * The unpack version is ignored.
     */
    snprintf(SQL,MAXSQL,
        "SELECT ars_pk from %s where upload_fk='%s' and ars_success=TRUE",
           AgentARSName, Upload_Pk);
    result =  PQexec(pgConn, SQL);
    if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(101);

    if (PQntuples(result) > 0) /* if there is a value */
    {  
      PQclear(result);
      LOG_WARNING("Upload_pk %s, has already been unpacked.  No further action required", 
              Upload_Pk)
      SafeExit(0);
    }
    PQclear(result);

    /* write the unpack_ars start record */
    ars_pk = fo_WriteARS(pgConn, ars_pk, atoi(Upload_Pk), agent_pk, AgentARSName, 0, 0);

    /* Get Pfile path and Pfile_Pk, from Upload_Pk */
  snprintf(SQL,MAXSQL,
        "SELECT pfile.pfile_sha1 || '.' || pfile.pfile_md5 || '.' || pfile.pfile_size AS pfile, pfile_fk, pfile_size FROM upload INNER JOIN pfile ON upload.pfile_fk = pfile.pfile_pk WHERE upload.upload_pk = '%s'", 
           Upload_Pk);
    result =  PQexec(pgConn, SQL);
    if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(102);

    if (PQntuples(result) > 0) /* if there is a value */
    {  
      Pfile = strdup(PQgetvalue(result,0,0));
      Pfile_Pk = strdup(PQgetvalue(result,0,1));
      Pfile_size = atol(PQgetvalue(result, 0, 2));
      if (Pfile_size == 0)
      {  
        PQclear(result);
        LOG_WARNING("Uploaded file (Upload_pk %s), is zero length.  There is nothing to unpack.", 
                      Upload_Pk)
        SafeExit(0);
      }

      PQclear(result);
    }

    // Determine if uploadtree records should go into a separate table.
    // If the input file size is > 500MB, then create a separate uploadtree_{upload_pk} table
    // that inherits from the master uploadtree table.
    // Save uploadtree_tablename, it will get written to upload.uploadtree_tablename later.
    if (Pfile_size > 500000000)
    {
      sprintf(uploadtree_tablename, "uploadtree_%s", Upload_Pk);
      if (!fo_tableExists(pgConn, uploadtree_tablename))
      {
        snprintf(SQL,MAXSQL,"CREATE TABLE %s (LIKE uploadtree INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES); ALTER TABLE %s ADD CONSTRAINT %s CHECK (upload_fk=%s); ALTER TABLE %s INHERIT uploadtree", 
               uploadtree_tablename, uploadtree_tablename, uploadtree_tablename, Upload_Pk, uploadtree_tablename);
        PQsetNoticeProcessor(pgConn, SQLNoticeProcessor, SQL);  // ignore notice about implicit primary key index creation
        result =  PQexec(pgConn, SQL);
        // Ignore postgres notice about creating an implicit index
        if (PQresultStatus(result) != PGRES_NONFATAL_ERROR)
          if (fo_checkPQcommand(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(103);
        PQclear(result);
      }
    }
    else
      strcpy(uploadtree_tablename, "uploadtree_a");

  }

  CheckCommands(Quiet);
  if (NewDir) MkDir(NewDir);
  if (Verbose) { fclose(stderr) ; stderr=stdout; } /* don't interlace! */
  if (ListOutName != NULL)
  {
    if ((ListOutName[0]=='-') && (ListOutName[1]=='\0'))
      ListOutFile=stdout;
    else ListOutFile = fopen(ListOutName,"w");
    if (!ListOutFile)
    {
      LOG_ERROR("pfile %s Unable to write to %s\n",Pfile_Pk,ListOutName)
      SafeExit(104);
    }
    else
    {
      /* Start the file */
      fputs("<xml tool=\"ununpack\" ",ListOutFile);
      fputs("version=\"",ListOutFile);
      fputs(Version,ListOutFile);
      fputs("\" ",ListOutFile);
      fputs("compiled_date=\"",ListOutFile);
      fputs(__DATE__,ListOutFile);
      fputs(" ",ListOutFile);
      fputs(__TIME__,ListOutFile);
      fputs("\"",ListOutFile);
      fputs(">\n",ListOutFile);
    }
    /* Problem: When parallel processing, the XML may be generated out
	   of order.  Solution?  When using XML, only use 1 thread. */
    MaxThread=1;
  }

  // Set ReunpackSwitch if the uploadtree records are missing from the database.
  if (!ReunpackSwitch && UseRepository)
  {
    snprintf(SQL,MAXSQL,"SELECT uploadtree_pk FROM uploadtree WHERE upload_fk=%s limit 1;",Upload_Pk);
    result =  PQexec(pgConn, SQL);
    if (fo_checkPQresult(pgConn, result, SQL, __FILE__, __LINE__)) SafeExit(105);
    if (PQntuples(result) == 0) ReunpackSwitch=1;
    PQclear(result);
  }

  /*** process files from command line ***/
  for( ; optind<argc; optind++)
  {
    CksumFile *CF=NULL;
    Cksum *Sum;
    int i;
    if (Fname) { free(Fname); Fname=NULL; }
    if (ListOutName != NULL)
    {
      fprintf(ListOutFile,"<source source=\"%s\" ",argv[optind]);
      if (UseRepository && !fo_RepExist(REP_FILES,argv[optind]))
      {
        /* make sure the source exists in the src repository */
        if (fo_RepImport(argv[optind],REP_FILES,argv[optind],1) != 0)
        {
          LOG_ERROR("Failed to import '%s' as '%s' into the repository",argv[optind],argv[optind])
          SafeExit(106);
        }
      }
    }

    if (UseRepository)
    {
      if (fo_RepExist(REP_FILES,argv[optind]))
      {
        Fname=fo_RepMkPath(REP_FILES,argv[optind]);
      }
      else if (fo_RepExist(REP_GOLD,argv[optind]))
      {
        Fname=fo_RepMkPath(REP_GOLD,argv[optind]);
        if (fo_RepImport(Fname,REP_FILES,argv[optind],1) != 0)
        {
          LOG_ERROR("Failed to import '%s' as '%s' into the repository",Fname,argv[optind])
          SafeExit(107);
        }
      }

      if (Fname)
      {
        FnameCheck = Fname;
        CF = SumOpenFile(Fname);
      }
      else
      {
        LOG_ERROR("NO file unpacked.  File %s does not exist either in GOLD or FILES", Pfile);
        SafeExit(108);
      }
      /* else: Fname is NULL and CF is NULL */
    }
    else 
    {
      FnameCheck = argv[optind];
      CF = SumOpenFile(argv[optind]);
    }

    /* Check file to unpack.  Does it exist?  Is it zero length? */
    if (stat(FnameCheck,&Stat)) 
    {
      LOG_ERROR("File to unpack is unavailable: %s, error: %s", Fname, strerror(errno));
      SafeExit(109);
    }
    else
    if (Stat.st_size < 1)
    {
      LOG_WARNING("File to unpack is empty: %s", Fname);
      SafeExit(110);
    }

    if (ListOutFile)
    {
      if (CF)
      {
        Sum = SumComputeBuff(CF);
        SumCloseFile(CF);
        if (Sum)
        {
          fputs("fuid=\"",ListOutFile);
          for(i=0; i<20; i++)
          { fprintf(ListOutFile,"%02X",Sum->SHA1digest[i]); }
          fputs(".",ListOutFile);
          for(i=0; i<16; i++)
          { fprintf(ListOutFile,"%02X",Sum->MD5digest[i]); }
          fputs(".",ListOutFile);
          fprintf(ListOutFile,"%Lu",(long long unsigned int)Sum->DataLen);
          fputs("\" ",ListOutFile);
          free(Sum);
        } /* if Sum */
      } /* if CF */
      else /* file too large to mmap (probably) */
      {
        FILE *Fin;
        Fin = fopen(argv[optind],"rb");
        if (Fin)
        {
          Sum = SumComputeFile(Fin);
          if (Sum)
          {
            fputs("fuid=\"",ListOutFile);
            for(i=0; i<20; i++)
            { fprintf(ListOutFile,"%02X",Sum->SHA1digest[i]); }
            fputs(".",ListOutFile);
            for(i=0; i<16; i++)
            { fprintf(ListOutFile,"%02X",Sum->MD5digest[i]); }
            fputs(".",ListOutFile);
            fprintf(ListOutFile,"%Lu",(long long unsigned int)Sum->DataLen);
            fputs("\" ",ListOutFile);
            free(Sum);
          }
          fclose(Fin);
        }
      } /* else no CF */
      fprintf(ListOutFile,">\n"); /* end source XML */
    }
    if (Fname)	TraverseStart(Fname,"called by main via args",NewDir,Recurse);
    else		TraverseStart(argv[optind],"called by main",NewDir,Recurse);
    if (ListOutName != NULL) fprintf(ListOutFile,"</source>\n");
  } /* end for */

  /* free memory */
  if (Fname) { free(Fname); Fname=NULL; }

  /* process pfile from scheduler */
  if (Pfile)
  {
    if (0 == (rvExist1 = fo_RepExist2(REP_FILES,Pfile)))
    {
      Fname=fo_RepMkPath(REP_FILES,Pfile);
    }
    else if (0 == (rvExist2 = fo_RepExist2(REP_GOLD,Pfile)))
    {
      Fname=fo_RepMkPath(REP_GOLD,Pfile);
      if (fo_RepImport(Fname,REP_FILES,Pfile,1) != 0)
      {
        LOG_ERROR("Failed to import '%s' as '%s' into the repository",Fname,Pfile)
        SafeExit(111);
      }
    }
    if (Fname)
    {
      TraverseStart(Fname,"called by main via env",NewDir,Recurse);
      free(Fname);
      Fname=NULL;
    }
    else
    {
      LOG_ERROR("NO file unpacked!");
      if (rvExist1 > 0)
      {
        Fname=fo_RepMkPath(REP_FILES, Pfile);
        LOG_ERROR("Error is %s for %s", strerror(rvExist1), Fname);
      }
      if (rvExist2 > 0)
      {
        Fname=fo_RepMkPath(REP_GOLD, Pfile);
        LOG_ERROR("Error is %s for %s", strerror(rvExist2), Fname);
      }
      SafeExit(112);
    }
  }

  /* recurse on all the children */
  if (Thread > 0) do
  {
    Pid = ParentWait();
    Thread--;
    if (Pid >= 0)
    {
      if (!Queue[Pid].ChildEnd)
      {
        /* copy over data */
        if (Recurse > 0)
          Traverse(Queue[Pid].ChildRecurse,NULL,"called by wait",NULL,Recurse-1,&Queue[Pid].PI);
        else if (Recurse < 0)
          Traverse(Queue[Pid].ChildRecurse,NULL,"called by wait",NULL,Recurse,&Queue[Pid].PI);
      }
    }
  } while(Pid >= 0);

  if (MagicCookie) magic_close(MagicCookie);
  if (ListOutFile)
  {
    fprintf(ListOutFile,"<summary files_regular=\"%d\" files_compressed=\"%d\" artifacts=\"%d\" directories=\"%d\" containers=\"%d\" />\n",
        TotalFiles,TotalCompressedFiles,TotalArtifacts,
        TotalDirectories,TotalContainers);
    fputs("</xml>\n",ListOutFile);
  }
  if (pgConn)
  {
    /* If it completes, mark it! */
    if (Upload_Pk)
    {
      snprintf(SQL,MAXSQL,"UPDATE upload SET upload_mode = (upload_mode | (1<<5)), uploadtree_tablename='%s' WHERE upload_pk = '%s';",uploadtree_tablename, Upload_Pk);
      result =  PQexec(pgConn, SQL); /* UPDATE upload */
      if (fo_checkPQcommand(pgConn, result, SQL, __FILE__ ,__LINE__)) SafeExit(113);
      PQclear(result);

      snprintf(SQL,MAXSQL,"UPDATE %s SET realparent = getItemParent(uploadtree_pk) WHERE upload_fk = '%s'",uploadtree_tablename, Upload_Pk);
      result =  PQexec(pgConn, SQL); /* UPDATE uploadtree */
      if (fo_checkPQcommand(pgConn, result, SQL, __FILE__ ,__LINE__)) SafeExit(114);
      PQclear(result);
    }

    if (ars_pk) fo_WriteARS(pgConn, ars_pk, atoi(Upload_Pk), agent_pk, AgentARSName, 0, 1);
  }
  if (ListOutFile && (ListOutFile != stdout))
  {
    fclose(ListOutFile);
  }

  if (UnlinkAll && MaxThread > 1)
  {
    /* Delete temporary files */
    if (strcmp(NewDir, ".")) RemoveDir(NewDir);
  }
 
  SafeExit(0);
  return(0);  // never executed but makes the compiler happy
} 
コード例 #9
0
ファイル: main.c プロジェクト: gerv/fossology
/**
 * @brief main function for the copyright agent
 *
 * The copyright agent is used to automatically locate copyright statements
 * found in code.
 *
 * There are 3 ways to use the copyright agent:
 *   1. Command Line Analysis :: test a file from the command line
 *   2. Agent Based Analysis  :: waits for commands from stdin
 *   3. Accuracy Test         :: tests the accuracy of the copyright agent
 *
 * +-----------------------+
 * | Command Line Analysis |
 * +-----------------------+
 *
 * To analyze a file from the command line:
 *   -C <filename>      :: run copyright agent from command line
 *   -d                 :: turn on debugging information
 *   -T <Copyright Statements | URLs| Emails> :: Copyright Statements | URLs |Emails
 *
 *   example:
 *     $ ./copyright -C myfiletoscan
 *
 * +----------------------+
 * | Agent Based Analysis |
 * +----------------------+
 *
 * To run the copyright agent as an agent simply run with no command line args
 *   -i                 :: initialize a connection to the database
 *   -d                 :: turn on debugging information
 *
 *   example:
 *     $ upload_pk | ./copyright
 *
 * +---------------+
 * | Accuracy Test |
 * +---------------+
 *
 * To test the accuracy of the copyright agent run with a -t. Make sure to run the
 * accuracy tests in the source directory with the testdata directory:
 *   -t                 :: run the accuracy analysis
 *
 *   example:
 *     $ ./copyright -t
 *
 * Running the tests will create 3 files:
 * 1. Matches: contains all of the matches found by the copyright agent, information
 *             includes what file the match was found in, the dictionary element
 *             that it matched, the name that it matched and the text that was found
 * 2. False_Positives: contains all of the false positives found by the agent,
 *             information in the file includes the file the false positive was
 *             in, the dictionary match, the name match, and the text
 * 3. Flase_Negatives: contains all of the false negatives found by the agent,
 *             information in the file includes the file the false negative was
 *             in, and the text of the false negative
 *
 * NOTE: -d will produces the exact same style of Matches file that the accuracy
 *       testing does. Currently this is the only thing that -d will produce
 *
 * @param argc the number of command line arguments
 * @param argv the command line arguments
 * @return 0 on a successful program execution
 */
int main(int argc, char** argv)
{
  /* primitives */
  char sql[512];                // buffer for database access
  int c, i = -1;                // temporary int containers
  int num_files = 0;            // the number of rows in a job
  int ars_pk = 0;               // the args primary key
  int user_pk = 0;
  long upload_pk = 0;           // the upload primary key
  long agent_pk = 0;            // the agents primary key
  char *SVN_REV = NULL;
  char *VERSION = NULL;
  char agent_rev[myBUFSIZ];
  char copy_buf[FILENAME_MAX];
  char name_buf[FILENAME_MAX];
  int report_type = 7;         // defaul as all. binary xxx 1st number as email, 2nd number as url, 3rd number as statement 
  int cli_run = 0;               // when run from command line, that mean -C option is set; 1: yes, 0: no

  /* Database structs */
  PGconn* pgConn = NULL;        // the connection to Database
  PGresult* pgResult = NULL;    // result of a database access

  /* copyright structs */
  copyright copy;               // the work horse of the copyright agent
  pair curr;                    // pair to push into the file list

  /* verbose data */
  FILE* mout = NULL;

  /* set the output streams */
  cout = stdout;
  cerr = stdout;
  cin = stdin;

  /* connect to the scheduler */
  fo_scheduler_connect(&argc, argv, &pgConn);

  /* initialize complex data strcutres */
  memset(copy_buf, '\0', sizeof(copy_buf));
  memset(name_buf, '\0', sizeof(copy_buf));
  snprintf(copy_buf, sizeof(copy_buf),
      "%s/mods-enabled/copyright/agent/copyright.dic",
      sysconfigdir);
  snprintf(name_buf, sizeof(name_buf),
      "%s/mods-enabled/copyright/agent/names.dic",
      sysconfigdir);

  if(!copyright_init(&copy, copy_buf, name_buf))
  {
    fprintf(cerr, "FATAL %s.%d: copyright initialization failed\n", __FILE__, __LINE__);
    fprintf(cerr, "FATAL %s\n", strerror(errno));
    fflush(cerr);
    return 1;
  }

  /* parse the command line options */
  while((c = getopt(argc, argv, "T:dc:C:tiVvh")) != -1)
  {
    switch(c)
    {
      case 'v': /* debugging */
        mout = fopen("Matches", "w");
        if(!mout)
        {
          fprintf(cerr, "ERROR could not open Matches for logging\n");
          fflush(cerr);
        }
        else
        {
          verbose = 1;
        }
        break;
      case 'C': /* run from command line */
        cli_run = 1;
        pair_init(&curr, string_function_registry(), int_function_registry());

        pair_set_first(curr, optarg);
        pair_set_second(curr, &i);
        num_files++;

        break;
      case 'T': /* report type, Copyright Statements | URLs| Emails */
        report_type = atoi(optarg);
        printf("report_type is:%d\n", report_type);
        break;
      case 't': /* run accuracy testing */
        run_test_files(copy);
        copyright_destroy(copy);
        return 0;
      case 'i': /* initialize database connections */
        copyright_destroy(copy);
        PQfinish(pgConn);
        return 0;
      case 'V':
        printf("%s", BuildVersion);
        copyright_destroy(copy);
        PQfinish(pgConn);
	return 0;
      default: /* error, print usage */
        copyright_usage(argv[0]);
        return 3;
    }
  }

  /** run from command line */
  if (1 == cli_run) {
    perform_analysis(pgConn, copy, curr, agent_pk, mout, report_type);
    pair_destroy(curr);
  }

  /* if there are no files in the file list then the agent is begin run from */
  /* the scheduler, open the database and grab the files to be analyzed      */
  if(num_files == 0)
  {
    /* create the sql copy structure */
    sqlcpy = fo_sqlCopyCreate(pgConn, "copyright", 32768, 7,
        "agent_fk", "pfile_fk", "copy_startbyte", "copy_endbyte", "content", "hash", "type");

    /* book keeping */
    pair_init(&curr, string_function_registry(), int_function_registry());
    db_connected = 1;
    SVN_REV = fo_sysconfig("copyright", "SVN_REV");
    VERSION = fo_sysconfig("copyright", "VERSION");
    sprintf(agent_rev, "%s.%s", VERSION, SVN_REV);
    agent_pk = fo_GetAgentKey(pgConn, AGENT_NAME, 0, agent_rev, AGENT_DESC);

    /* make sure that we are connected to the database */
    if(!check_copyright_table(pgConn))
    {
      return 5;
    }

    user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */

    /* enter the main agent loop */
    while(fo_scheduler_next())
    {
      upload_pk = atol(fo_scheduler_current());

      /* Check Permissions */
      if (GetUploadPerm(pgConn, upload_pk, user_pk) < PERM_WRITE)
      {
        LOG_ERROR("You have no update permissions on upload %ld", upload_pk);
        continue;
      }
                                               
      ars_pk = fo_WriteARS(pgConn, 0, upload_pk, agent_pk, AGENT_ARS, NULL, 0);

      sprintf(sql, fetch_pfile, upload_pk, agent_pk, agent_pk);
      pgResult = PQexec(pgConn, sql);
      num_files = PQntuples(pgResult);

      for(i = 0; i < num_files; i++)
      {
        c = atoi(PQgetvalue(pgResult, i, PQfnumber(pgResult, "pfile_pk")));
        pair_set_first(curr, PQgetvalue(pgResult, i, PQfnumber(pgResult, "pfilename")));
        pair_set_second(curr, &c);
        perform_analysis(pgConn, copy, curr, agent_pk, mout, REPORTALL);
      }

      fo_WriteARS(pgConn, ars_pk, upload_pk, agent_pk, AGENT_ARS, NULL, 1);
      PQclear(pgResult);
    }

    pair_destroy(curr);
  }

  if(db_connected)
  {
    fo_sqlCopyDestroy(sqlcpy, 1);
    PQfinish(pgConn);
  }

  if(verbose)
  {
    fclose(mout);
  }

  copyright_destroy(copy);
  fo_scheduler_disconnect(0);

  return 0;
}
コード例 #10
0
int main  (int argc, char *argv[])
{
    int arg;
    char *Parm = NULL;
    char *TempFileDir=NULL;
    int c;
    int InitFlag=0;
    int CmdlineFlag = 0; /** run from command line flag, 1 yes, 0 not */
    int user_pk;
    char *agent_desc = "Network downloader.  Uses wget(1).";

    memset(GlobalTempFile,'\0',MAXCMD);
    memset(GlobalURL,'\0',MAXCMD);
    memset(GlobalParam,'\0',MAXCMD);
    memset(GlobalType,'\0',MAXCMD);
    GlobalUploadKey = -1;
    int upload_pk = 0;           // the upload primary key
    //int Agent_pk;
    char *SVN_REV;
    char *VERSION;
    char agent_rev[MAXCMD];

    /* open the connection to the scheduler and configuration */
    fo_scheduler_connect(&argc, argv, &pgConn);

    /* Process command-line */
    while((c = getopt(argc,argv,"d:Gg:ik:A:R:l:Cc:Vvh")) != -1)
    {
        switch(c)
        {
        case 'd':
            TempFileDir = PathCheck(optarg);
            break;
        case 'g':
        {
            struct group *SG;
            SG = getgrnam(optarg);
            if (SG) ForceGroup = SG->gr_gid;
        }
        break;
        case 'G':
            GlobalImportGold=0;
            break;
        case 'i':
            InitFlag=1;
            break;
        case 'k':
            GlobalUploadKey = atol(optarg);
            if (!GlobalTempFile[0])
                strcpy(GlobalTempFile,"wget.default_download");
            break;
        case 'A':
            sprintf(GlobalParam, "%s -A %s ",GlobalParam, optarg);
            break;
        case 'R':
            sprintf(GlobalParam, "%s -R %s ",GlobalParam, optarg);
            break;
        case 'l':
            sprintf(GlobalParam, "%s -l %s ",GlobalParam, optarg);
            break;
        case 'c':
            break; /* handled by fo_scheduler_connect() */
        case 'C':
            CmdlineFlag = 1;
            break;
        case 'v':
            break;
        case 'V':
            printf("%s", BuildVersion);
            SafeExit(0);
        default:
            Usage(argv[0]);
            SafeExit(-1);
        }
    }
    if (argc - optind > 1)
    {
        Usage(argv[0]);
        SafeExit(-1);
    }

    /* When initializing the DB, don't do anything else */
    if (InitFlag)
    {
        if (pgConn) PQfinish(pgConn);
        SafeExit(0);
    }

    SVN_REV = fo_sysconfig("wget_agent", "SVN_REV");
    VERSION = fo_sysconfig("wget_agent", "VERSION");
    sprintf(agent_rev, "%s.%s", VERSION, SVN_REV);
    /* Get the Agent Key from the DB */
    fo_GetAgentKey(pgConn, basename(argv[0]), GlobalUploadKey, agent_rev, agent_desc);

    /** get proxy */
    GetProxy();

    /* Run from the command-line (for testing) */
    for(arg=optind; arg < argc; arg++)
    {
        memset(GlobalURL,'\0',sizeof(GlobalURL));
        strncpy(GlobalURL,argv[arg],sizeof(GlobalURL));
        /* If the file contains "://" then assume it is a URL.
           Else, assume it is a file. */
        LOG_VERBOSE0("Command-line: %s",GlobalURL);
        if (strstr(GlobalURL,"://"))
        {
            fo_scheduler_heart(1);
            LOG_VERBOSE0("It's a URL");
            if (GetURL(GlobalTempFile,GlobalURL,TempFileDir) != 0)
            {
                LOG_FATAL("Download of %s failed.",GlobalURL);
                SafeExit(21);
            }
            if (GlobalUploadKey != -1) {
                DBLoadGold();
            }
            unlink(GlobalTempFile);
        }
        else /* must be a file */
        {
            LOG_VERBOSE0("It's a file -- GlobalUploadKey = %ld",GlobalUploadKey);
            if (GlobalUploadKey != -1)
            {
                memcpy(GlobalTempFile,GlobalURL,MAXCMD);
                DBLoadGold();
            }
        }
    }

    /* Run from scheduler! */
    if (0 == CmdlineFlag)
    {
        user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */
        while(fo_scheduler_next())
        {
            Parm = fo_scheduler_current(); /* get piece of information, including upload_pk, downloadfile url, and parameters */
            if (Parm && Parm[0])
            {
                fo_scheduler_heart(1);
                /* set globals: uploadpk, downloadfile url, parameters */
                SetEnv(Parm,TempFileDir);
                upload_pk = GlobalUploadKey;

                /* Check Permissions */
                if (GetUploadPerm(pgConn, upload_pk, user_pk) < PERM_WRITE)
                {
                    LOG_ERROR("You have no update permissions on upload %d", upload_pk);
                    continue;
                }

                char TempDir[MAXCMD];
                memset(TempDir,'\0',MAXCMD);
                snprintf(TempDir, MAXCMD-1, "%s/wget", TempFileDir); // /var/local/lib/fossology/agents/wget
                struct stat Status;

                if (GlobalType[0])
                {
                    if (GetVersionControl() == 0)
                    {
                        DBLoadGold();
                        unlink(GlobalTempFile);
                    }
                    else
                    {
                        LOG_FATAL("upload %ld File retrieval failed: uploadpk=%ld tempfile=%s URL=%s Type=%s",
                                  GlobalUploadKey,GlobalUploadKey,GlobalTempFile,GlobalURL, GlobalType);
                        SafeExit(23);
                    }
                }
                else if (strstr(GlobalURL, "*") || stat(GlobalURL, &Status) == 0)
                {
                    if (!Archivefs(GlobalURL, GlobalTempFile, TempFileDir, Status))
                    {
                        LOG_FATAL("Failed to archieve. GlobalURL, GlobalTempFile, TempFileDir are: %s, %s, %s, "
                                  "Mode is: %lo (octal)\n", GlobalURL, GlobalTempFile, TempFileDir, (unsigned long) Status.st_mode);
                        SafeExit(50);
                    }
                    DBLoadGold();
                    unlink(GlobalTempFile);
                }
                else
                {
                    if (GetURL(GlobalTempFile,GlobalURL,TempDir) == 0)
                    {
                        DBLoadGold();
                        unlink(GlobalTempFile);
                    }
                    else
                    {
                        LOG_FATAL("upload %ld File retrieval failed: uploadpk=%ld tempfile=%s URL=%s",
                                  GlobalUploadKey,GlobalUploadKey,GlobalTempFile,GlobalURL);
                        SafeExit(22);
                    }
                }
            }
        }
    } /* if run from scheduler */

    SafeExit(0);
    exit(0);  /* to prevent compiler warning */
} /* main() */