/**
* \brief for function TaintURL
*/
void testTaintURL()
{
char Sin[MAX_LENGTH];
char Sout[MAX_LENGTH];
int SoutSize = MAX_LENGTH;
/* the URL is failed to taint*/
strcpy(Sin, "http://fossology.org #");
int result = TaintURL(Sin, Sout, SoutSize);
CU_ASSERT_EQUAL(result, 0); /* failed to taint */
/* the URL is tainted */
strcpy(Sin, "http://fossology.org/`debian/ 1.0.0/");
result = TaintURL(Sin, Sout, SoutSize);
CU_ASSERT_EQUAL(result, 1); /* tainted */
#if 0
#endif
}
/*********************************************************
GetURL(): Do the wget.
*********************************************************/
int GetURL (char *TempFile, char *URL, char *TempFileDir)
{
char CMD[MAXCMD];
char TaintedURL[MAXCMD];
char TmpLine[256];
int rc;
FILE *Fin;
#if 1
char WgetArgs[]="--no-check-certificate --progress=dot -rc -np -e robots=off -k";
#else
/* wget < 1.10 does not support "--no-check-certificate" */
char WgetArgs[]="--progress=dot";
#endif
if (!TaintURL(URL,TaintedURL,MAXCMD))
{
printf("FATAL: Failed to parse the URL\n");
printf("LOG: Failed to taint the URL '%s'\n",URL);
fflush(stdout);
DBclose(DB);
exit(10);
}
memset(CMD,'\0',MAXCMD);
/***
Wget options:
--progress=dot :: display a new line as it progresses.
--no-check-certificate :: download HTTPS files even if the cert cannot
be validated. (Neal has many issues with SSL and does not view it
as very secure.) Without this, some caching proxies and web sites
with old certs won't download. Granted, in theory a bad cert should
prevent downloads. In reality, 99.9% of bad certs are because the
admin did not notice that they expired and not because of a hijacking
attempt.
***/
struct stat sb;
int rc_system =0;
/* Run from scheduler! delete the temp directory, /var/local/lib/fossology/agents/wget */
if (!stat(TempFileDir, &sb) && TempFile && TempFile[0])
{
memset(CMD,'\0',MAXCMD);
snprintf(CMD,MAXCMD-1, "rm -rf '%s' 2>&1", TempFileDir);
rc_system = system(CMD);
if (rc_system != 0) exit(23); // failed to delete the temperary directory
}
if (TempFile && TempFile[0])
{
/* Delete the temp file if it exists */
unlink(TempFile);
snprintf(CMD,MAXCMD-1,". %s ; /usr/bin/wget %s -P '%s' '%s' %s 2>&1",
PROXYFILE,WgetArgs,TempFileDir,TaintedURL,GlobalParam);
}
else if(TempFileDir && TempFileDir[0])
{
snprintf(CMD,MAXCMD-1,". %s ; /usr/bin/wget %s -P '%s' '%s' %s 2>&1",
PROXYFILE,WgetArgs, TempFileDir, TaintedURL, GlobalParam);
}
else
{
snprintf(CMD,MAXCMD-1,". %s ; /usr/bin/wget %s '%s' %s 2>&1",
PROXYFILE,WgetArgs,TaintedURL, GlobalParam);
}
Fin = popen(CMD,"r");
if (!Fin)
{
printf("FATAL upload %ld Failed to retrieve file.\n",GlobalUploadKey);
printf("LOG upload %ld Failed to run command: %s\n",GlobalUploadKey,CMD);
fflush(stdout);
DBclose(DB);
exit(11);
}
while(ReadLine(Fin,TmpLine,256) != -1)
{
/* Track if a line is read.
If this does not change after a minute, then heartbeat will
not display. This catches cases where wget hangs. */
InitHeartbeat();
}
InitHeartbeat();
rc = pclose(Fin); /* rc is the exit status */
if (WIFEXITED(rc) && (WEXITSTATUS(rc) != 0))
{
printf("ERROR upload %ld Download failed\n",GlobalUploadKey);
printf("LOG upload %ld Download failed; Return code %d from: %s\n",GlobalUploadKey,WEXITSTATUS(rc),CMD);
fflush(stdout);
unlink(GlobalTempFile);
DBclose(DB);
exit(12);
}
if (WIFEXITED(rc) && WIFSIGNALED(rc))
{
printf("ERROR upload %ld Download killed by a signal\n",GlobalUploadKey);
printf("LOG upload %ld Download killed by signal %d\n",GlobalUploadKey,WTERMSIG(rc));
fflush(stdout);
unlink(GlobalTempFile);
DBclose(DB);
exit(13);
}
if (WIFEXITED(rc) && WIFSIGNALED(rc))
{
printf("ERROR upload %ld Download killed by a signal\n",GlobalUploadKey);
printf("LOG upload %ld Download killed by signal %d\n",GlobalUploadKey,WTERMSIG(rc));
fflush(stdout);
unlink(GlobalTempFile);
DBclose(DB);
exit(14);
}
/* Run from scheduler! store /var/local/lib/fossology/agents/wget/../<files|directories> to one temp file */
if (TempFile && TempFile[0])
{
char TempFilePath[MAXCMD];
memset(TempFilePath,'\0',MAXCMD);
/* for one url http://a.org/test.deb, TempFilePath should be /var/local/lib/fossology/agents/wget/a.org/test.deb */
int Position = GetPosition(TaintedURL);
if (0 == Position) exit(26);
snprintf(TempFilePath, MAXCMD-1, "%s/%s", TempFileDir, TaintedURL + Position);
if (!stat(TempFilePath, &sb))
{
memset(CMD,'\0',MAXCMD);
if (S_ISDIR(sb.st_mode))
{
snprintf(CMD,MAXCMD-1, "find '%s' -mindepth 1 -type d -empty -exec rmdir {} \\; > /dev/null 2>&1", TempFilePath);
system(CMD); // delete all empty directories downloaded
memset(CMD,'\0',MAXCMD);
snprintf(CMD,MAXCMD-1, "tar -cvvf '%s' -C '%s' ./ 2>&1", TempFile, TempFilePath);
}
else
{
snprintf(CMD,MAXCMD-1, "mv '%s' '%s' 2>&1", TempFilePath, TempFile);
}
rc_system = system(CMD);
if (rc_system != 0) exit(24); // failed to store the temperary directory(one file) as one temperary file
}
}
if (TempFile && TempFile[0] && !IsFile(TempFile,1))
{
printf("ERROR upload %ld File %s not created from %s\n",GlobalUploadKey,TempFile,URL);
printf("LOG upload %ld File not created from command: %s\n",GlobalUploadKey,CMD);
fflush(stdout);
DBclose(DB);
exit(15);
}
printf("LOG upload %ld Downloaded %s to %s\n",GlobalUploadKey,URL,TempFile);
return(0);
} /* GetURL() */