Beispiel #1
0
/**
 * \brief for function TaintURL 
 */
void testTaintURL()
{
  char Sin[MAX_LENGTH];
  char Sout[MAX_LENGTH];
  int SoutSize = MAX_LENGTH;
  /* the URL is failed to taint*/
  strcpy(Sin, "http://fossology.org #");
  int result = TaintURL(Sin, Sout, SoutSize);
  CU_ASSERT_EQUAL(result, 0); /* failed to taint */
  /* the URL is tainted */
  strcpy(Sin, "http://fossology.org/`debian/ 1.0.0/");
  result = TaintURL(Sin, Sout, SoutSize);
  CU_ASSERT_EQUAL(result, 1); /*  tainted */
#if 0
#endif
}
Beispiel #2
0
/*********************************************************
 GetURL(): Do the wget.
 *********************************************************/
int	GetURL	(char *TempFile, char *URL, char *TempFileDir)
{
  char CMD[MAXCMD];
  char TaintedURL[MAXCMD];
  char TmpLine[256];
  int rc;
  FILE *Fin;
#if 1
  char WgetArgs[]="--no-check-certificate --progress=dot -rc -np -e robots=off -k";
#else
  /* wget < 1.10 does not support "--no-check-certificate" */
  char WgetArgs[]="--progress=dot";
#endif

  if (!TaintURL(URL,TaintedURL,MAXCMD))
	{
	printf("FATAL: Failed to parse the URL\n");
	printf("LOG: Failed to taint the URL '%s'\n",URL);
	fflush(stdout);
	DBclose(DB);
	exit(10);
	}

  memset(CMD,'\0',MAXCMD);
  /***
   Wget options:
   --progress=dot :: display a new line as it progresses.
   --no-check-certificate :: download HTTPS files even if the cert cannot
     be validated.  (Neal has many issues with SSL and does not view it
     as very secure.)  Without this, some caching proxies and web sites
     with old certs won't download.  Granted, in theory a bad cert should
     prevent downloads.  In reality, 99.9% of bad certs are because the
     admin did not notice that they expired and not because of a hijacking
     attempt.
   ***/
 
  struct stat sb;
  int rc_system =0;

  /* Run from scheduler! delete the temp directory, /var/local/lib/fossology/agents/wget */
  if (!stat(TempFileDir, &sb) && TempFile && TempFile[0])
  {
    memset(CMD,'\0',MAXCMD);
    snprintf(CMD,MAXCMD-1, "rm -rf '%s' 2>&1", TempFileDir);
    rc_system = system(CMD);
    if (rc_system != 0) exit(23); // failed to delete the temperary directory
     
  }

  if (TempFile && TempFile[0])
  {
    /* Delete the temp file if it exists */
    unlink(TempFile);
    snprintf(CMD,MAXCMD-1,". %s ; /usr/bin/wget %s -P '%s' '%s' %s 2>&1",
        PROXYFILE,WgetArgs,TempFileDir,TaintedURL,GlobalParam);
  }
  else if(TempFileDir && TempFileDir[0])
  {
    snprintf(CMD,MAXCMD-1,". %s ; /usr/bin/wget %s -P '%s' '%s' %s 2>&1",
      PROXYFILE,WgetArgs, TempFileDir, TaintedURL, GlobalParam);
  }
  else 
  {
    snprintf(CMD,MAXCMD-1,". %s ; /usr/bin/wget %s '%s' %s 2>&1",
      PROXYFILE,WgetArgs,TaintedURL, GlobalParam);
  }
  Fin = popen(CMD,"r");
  if (!Fin)
    {
    printf("FATAL upload %ld Failed to retrieve file.\n",GlobalUploadKey);
    printf("LOG upload %ld Failed to run command: %s\n",GlobalUploadKey,CMD);
    fflush(stdout);
    DBclose(DB);
    exit(11);
    }

  while(ReadLine(Fin,TmpLine,256) != -1)
	{
	/* Track if a line is read.
	   If this does not change after a minute, then heartbeat will
	   not display. This catches cases where wget hangs. */
	InitHeartbeat();
	}
  InitHeartbeat();

  rc = pclose(Fin);  /* rc is the exit status */
 
  if (WIFEXITED(rc) && (WEXITSTATUS(rc) != 0))
	{
	printf("ERROR upload %ld Download failed\n",GlobalUploadKey);
	printf("LOG upload %ld Download failed; Return code %d from: %s\n",GlobalUploadKey,WEXITSTATUS(rc),CMD);
	fflush(stdout);
	unlink(GlobalTempFile);
	DBclose(DB);
	exit(12);
	}

  if (WIFEXITED(rc) && WIFSIGNALED(rc))
	{
	printf("ERROR upload %ld Download killed by a signal\n",GlobalUploadKey);
	printf("LOG upload %ld Download killed by signal %d\n",GlobalUploadKey,WTERMSIG(rc));
	fflush(stdout);
	unlink(GlobalTempFile);
	DBclose(DB);
	exit(13);
	}

  if (WIFEXITED(rc) && WIFSIGNALED(rc))
	{
	printf("ERROR upload %ld Download killed by a signal\n",GlobalUploadKey);
	printf("LOG upload %ld Download killed by signal %d\n",GlobalUploadKey,WTERMSIG(rc));
	fflush(stdout);
	unlink(GlobalTempFile);
	DBclose(DB);
	exit(14);
	}

  /* Run from scheduler! store /var/local/lib/fossology/agents/wget/../<files|directories> to one temp file */
  if (TempFile && TempFile[0])
  {
    char TempFilePath[MAXCMD];
    memset(TempFilePath,'\0',MAXCMD);
    /* for one url http://a.org/test.deb, TempFilePath should be /var/local/lib/fossology/agents/wget/a.org/test.deb */
    int Position = GetPosition(TaintedURL);
    if (0 == Position) exit(26);
    snprintf(TempFilePath, MAXCMD-1, "%s/%s", TempFileDir, TaintedURL + Position);
    if (!stat(TempFilePath, &sb))
    {
      memset(CMD,'\0',MAXCMD);
      if (S_ISDIR(sb.st_mode))
      {
        snprintf(CMD,MAXCMD-1, "find '%s' -mindepth 1 -type d -empty -exec rmdir {} \\; > /dev/null 2>&1", TempFilePath);
        system(CMD); // delete all empty directories downloaded
        memset(CMD,'\0',MAXCMD);
        snprintf(CMD,MAXCMD-1, "tar -cvvf '%s' -C '%s' ./ 2>&1", TempFile, TempFilePath);
      }
      else
      {
        snprintf(CMD,MAXCMD-1, "mv '%s' '%s' 2>&1", TempFilePath, TempFile);
      }
      rc_system = system(CMD);
      if (rc_system != 0) exit(24); // failed to store the temperary directory(one file) as one temperary file
    }
  } 

  if (TempFile && TempFile[0] && !IsFile(TempFile,1))
	{
	printf("ERROR upload %ld File %s not created from %s\n",GlobalUploadKey,TempFile,URL);
	printf("LOG upload %ld File not created from command: %s\n",GlobalUploadKey,CMD);
	fflush(stdout);
	DBclose(DB);
	exit(15);
	}

  printf("LOG upload %ld Downloaded %s to %s\n",GlobalUploadKey,URL,TempFile);
  return(0);
} /* GetURL() */