static bool mapToFile(MaConn *conn, bool *rescan)
{
MaRequest *req;
MaResponse *resp;
req = conn->request;
resp = conn->response;
if (resp->filename == 0) {
resp->filename = makeFilename(conn, req->alias, req->url, 1);
}
req->dir = maLookupBestDir(req->host, resp->filename);
if (req->dir == 0) {
maFailRequest(conn, MPR_HTTP_CODE_NOT_FOUND, "Missing directory block for %s", resp->filename);
return 0;
}
mprAssert(req->dir);
req->auth = req->dir->auth;
if (!resp->fileInfo.valid && mprGetFileInfo(conn, resp->filename, &resp->fileInfo) < 0) {
#if UNUSED
if (req->method & (MA_REQ_GET | MA_REQ_POST)) {
maFailRequest(conn, MPR_HTTP_CODE_NOT_FOUND, "Can't open document: %s", resp->filename);
return 0;
}
#endif
}
if (resp->fileInfo.isDir) {
processDirectory(conn, rescan);
}
return 1;
}
void Index::createBase()
{
for (std::vector<std::string>::size_type i = 0; i < directories_.size(); ++i)
{
processDirectory(directories_.at(i), std::string());
}
}
bool ICOImageDecoder::decodeDirectory()
{
// Read and process directory.
if ((m_decodedOffset < sizeOfDirectory) && !processDirectory())
return false;
// Read and process directory entries.
return (m_decodedOffset >= (sizeOfDirectory + (m_dirEntries.size() * sizeOfDirEntry))) || processDirectoryEntries();
}
static int processDirectory(char *dirname, LinkedList *ll, int verbose) {
DIR *dd;
struct dirent *dent;
char *sp;
int len, status = 1;
char d[4096];
/*
* eliminate trailing slash, if there
*/
strcpy(d, dirname);
len = strlen(d);
if (len > 1 && d[len-1] == '/')
d[len-1] = '\0';
/*
* open the directory
*/
if ((dd = opendir(d)) == NULL) {
if (verbose)
fprintf(stderr, "Error opening directory `%s'\n", d);
return 1;
}
/*
* duplicate directory name to insert into linked list
*/
sp = strdup(d);
if (sp == NULL) {
fprintf(stderr, "Error adding `%s' to linked list\n", d);
status = 0;
goto cleanup;
}
if (!ll_add(ll, sp)) {
fprintf(stderr, "Error adding `%s' to linked list\n", sp);
free(sp);
status = 0;
goto cleanup;
}
if (len == 1 && d[0] == '/')
d[0] = '\0';
/*
* read entries from the directory
*/
while (status && (dent = readdir(dd)) != NULL) {
if (strcmp(".", dent->d_name) == 0 || strcmp("..", dent->d_name) == 0)
continue;
if (dent->d_type & DT_DIR) {
char b[4096];
sprintf(b, "%s/%s", d, dent->d_name);
status = processDirectory(b, ll, 0);
}
}
cleanup:
(void) closedir(dd);
return status;
}
NABoolean HHDFSTableStats::populate(struct hive_tbl_desc *htd)
{
// here is the basic outline how this works:
//
// 1. Walk SD descriptors of the table, one for the table
// itself and one for each partition. Each one represents
// one HDFS directory with files for the table.
// 2. For each list partition directory (or the directory for
// an unpartitioned table):
// 3. Walk through every file. For every file:
// 4. Determine bucket number (0 if file is not bucketed)
// 5. Add file to its bucket
// 6. Walk through blocks of file. For every block:
// 7. Get host list for this block and add it
// 9. Get file stats
// 10. Aggregate file stats for all files and buckets
// 11. Aggregate bucket stats for all buckets of the partition
// 12. Aggregate partition stats for all partitions of the table
NABoolean result = TRUE;
struct hive_sd_desc *hsd = htd->getSDs();
tableDir_ = hsd->location_;
numOfPartCols_ = htd->getNumOfPartCols();
recordTerminator_ = hsd->getRecordTerminator();
fieldTerminator_ = hsd->getFieldTerminator() ;
NAString hdfsHost;
Int32 hdfsPort = -1;
NAString tableDir;
while (hsd)
{
// split table URL into host, port and filename
splitLocation(hsd->location_, hdfsHost, hdfsPort, tableDir);
if (! connectHDFS(hdfsHost, hdfsPort))
CMPASSERT(fs_);
// put back fully qualified URI
tableDir = hsd->location_;
// visit the directory
result = processDirectory(tableDir, hsd->buckets_,
hsd->isTrulyText(),
hsd->getRecordTerminator(),
hsd->isSequenceFile());
hsd = hsd->next_;
}
disconnectHDFS();
validationJTimestamp_ = JULIANTIMESTAMP();
return result;
}
int main(int argc, char** argv) {
if( argc > 4 )
usage();
argc--;
argv++;
char* path = 0;
while( argc ) {
if( !strcmp(*argv, "-d") ) {
deleteFiles = true;
printf("will delete invalid files!\n");
} else {
if( !strcmp(*argv, "-v") ) {
verbose = true;
printf("will be verbose\n");
} else {
if( !strcmp(*argv, "-r") ) {
renameFiles = true;
printf("will rename invalid files!\n");
} else {
if( path != 0 ) {
free(path);
usage();
} else {
path = malloc(strlen(*argv));
strcpy(path, *argv);
}
}
}
}
argc--;
argv++;
}
if( deleteFiles && renameFiles ) {
printf("deleting and renaming is not possible at once\n");
usage();
}
if( path == 0 ) {
path = malloc(2);
strcpy(path, ".");
}
int found = processDirectory(path);
printf("found %d invalid files and directories\n", found);
free(path);
return 0;
}
void LoadFileListTask::processDirectory(const QString &path)
{
emit currentPathChanged(path);
QFileInfoList fileInfoList = QDir(path).entryInfoList(QDir::AllEntries | QDir::NoDotAndDotDot);
for (int j = 0; j < fileInfoList.size(); j++) {
QFileInfo fileInfo = fileInfoList.at(j);
if (fileInfo.isDir()) {
processDirectory(fileInfo.filePath());
}
else {
processFile(fileInfo.filePath());
}
}
}
static void processDirectory( void )
{
DIR *dirh;
struct dirent *dp;
dirstack *tmp;
if( RecurLevels != 0 ) {
extendPath( PathBuff, "*.*" );
dirh = opendir( PathBuff );
if( dirh != NULL ) {
--RecurLevels;
for( ;; ) {
if( DoneFlag ) return;
dp = readdir( dirh );
if( dp == NULL ) break;
#if defined( __WATCOMC__ ) && !defined( __UNIX__ )
if( !( dp->d_attr & _A_SUBDIR ) ) continue;
#else
{
struct stat sblk;
char tmp_path[_MAX_PATH+1];
strcpy( tmp_path, PathBuff );
extendPath( tmp_path, dp->d_name );
if( stat( tmp_path, &sblk ) == 0 && !S_ISDIR( sblk.st_mode ) ) {
continue;
}
}
#endif
if( dp->d_name[0] == '.' ) {
if( dp->d_name[1] == '.' || dp->d_name[1] == '\0' ) continue;
}
if( DoneFlag ) return;
tmp = (dirstack *) SafeMalloc( sizeof( dirstack ) );
extendPath( tmp->name, dp->d_name );
tmp->prev = Stack;
Stack = tmp;
processDirectory();
Stack = tmp->prev;
free( tmp );
}
++RecurLevels;
closedir( dirh );
}
}
executeWgrep();
}
static void nextWgrep( char **paths )
{
struct stat sblk;
Stack = (dirstack *) SafeMalloc( sizeof( dirstack ) );
while( *paths != NULL ) {
if( stat( *paths, &sblk ) == 0 && S_ISDIR( sblk.st_mode ) ) {
strcpy( Stack->name, *paths );
strcpy( CurrPattern, "*.*" );
} else {
splitPath( *paths, Stack->name, CurrPattern );
}
processDirectory( );
paths++;
}
free( Stack );
}
void Index::processDirectory(const lookupDirectory &baseDirectory, const std::string &relative)
{
std::string directory = baseDirectory.path + relative;
std::vector<file> content = directoryEntries(directory);
if (baseDirectory.withSubdirectories)
{
for (std::vector<file>::size_type i = 0; i < content.size(); ++i)
{
if (content.at(i).type == FileType::Directory)
{
processDirectory(baseDirectory, relative + content.at(i).name + "/");
}
}
}
processDesktopInDirectory(baseDirectory.path, relative, content);
processMimeApps(directory + "mimeapps.list");
}
/**
Parses all the files within the given directory (and within its subdirectories) and adds them to the database.
@param path: path to the directory to process
@param name: name of the directory to process
@param parseBody: boolean value stating whether the body must be parsed or not
*/
void DatasetGenerator::parseDirectory(string &path, const string &directory, bool parseBody)
{
aParseBody=parseBody;
//Open files where the times will be saved
string fileFullNameParsing="./Results/Parsing/parsingTimes_WB.txt";
string fileFullNameProcessing="./Results/Parsing/processingTimes_WB.txt";
string fileFullNameDirs="./Results/Parsing/dirNames_WB.txt";
parsingTimesFile.open(fileFullNameParsing.c_str(), ios::app);
processingTimesFile.open(fileFullNameProcessing.c_str(), ios::app);
dirNames.open(fileFullNameDirs.c_str(), ios::app);
//Parse directory
DB.connect();
processDirectory(path,directory);
DB.close();
//close files
parsingTimesFile.close();
processingTimesFile.close();
dirNames.close();
}
/**
Processes the given entity in order to know whether it is a directory or a file, an then call the corresponding method to process the entity.
@param path: path to the entity to process
@param name: name of the entity to process
*/
void DatasetGenerator::processEntity(string &path, struct dirent* entity)
{
//PROCESS DIRECTORIES
if(entity->d_type==DT_DIR)
{
//Do not process directory . and ..
if(entity->d_name[0]=='.'){
return;
}
processDirectory(path, string(entity->d_name));
return;
}
//PROCESS REGULAR FILES
if(entity->d_type==DT_REG)
{
processFile(path, string(entity->d_name));
return;
}
//PROCESS REST OF ENTITIES
cout <<"Not a file or directory: "<<entity->d_name<<endl;
}
int main(int argc, char** argv) {
int c;
while ((c = getopt(argc, argv, "vw")) != -1) {
switch (c) {
case 'v':
verbose = 1;
break;
case 'w':
sleepy = 1;
break;
default:
printf("Bad arguments\n");
printHelp();
return 1;
}
}
processExtensions();
/*if (exts) {
int i;
for (i = 0; *(exts + i); i++) {
printf("%s\n", *(exts + i));
}
}*/
processDirectory(argv);
if (exts) {
int i;
for (i = 0; *(exts + i); i++) {
free(*(exts + i));
}
}
free(exts);
return 0;
}
int main(int argc, char *argv[])
{
int app_result = 0;
QCoreApplication a(argc, argv);
QCoreApplication::setApplicationName("PatchMaker");
QCoreApplication::setApplicationVersion("1.0");
QStringList positionalArgs;
QStringList ignoreMasks;
if (commandLineArgumentsValid(&positionalArgs, &ignoreMasks))
{
processDirectory(positionalArgs.at(0),
positionalArgs.at(1),
positionalArgs.at(2),
ignoreMasks);
}
else
{
app_result = 1;
}
return app_result;
}
int
main(
int argc,
char *argv[]
)
{
int c; /* Command line character */
char * path; /* Path being processed */
me = argv[0];
add_ignore( "." );
add_ignore( ".." );
while(
(c = getopt( argc, argv, "aDc:dhFfgl:i:n:o:psuW:w" )) != EOF
) {
switch( c ) {
default:
fprintf( stderr, "%s: no -%c yet!\n", me, c );
/*FALLTHRU*/
case '?':
++nonfatal;
break;
case 'D':
++debug;
break;
case 'a':
++a_sw;
break;
case 'c':
whatColumn = atoi( optarg );
break;
case 'd':
++d_sw;
break;
case 'F':
++F_sw;
break;
case 'f':
++f_sw;
break;
case 'g':
combo_sw |= Iwanna_group;
break;
case 'h':
combo_sw |= Iwanna_size;
break;
case 'i':
add_ignore( optarg );
break;
case 'l':
depth = atoi( optarg );
break;
case 'p':
combo_sw |= Iwanna_perms;
break;
case 'o':
ofile = optarg;
break;
case 's':
++s_sw;
break;
case 'u':
combo_sw |= Iwanna_user;
break;
case 'W':
leadin = optarg;
Nleadin = strlen( leadin );
break;
case 'w':
++w_sw;
break;
}
}
if( nonfatal ) {
fprintf( stderr, "%s: illegal switch(es)\n", me );
fprintf( stderr,
"usage: %s "
"[-D] "
"[-a] "
"[-d] "
"[-c] "
"[-d depth] "
"[-f] "
"[-l limit] "
"[-i subdir] "
"[-s] "
"[-o ofile] "
"[-p] "
"[-W prefix] "
"[-w] "
"[dir...]"
"\n",
me
);
exit( 1 );
}
if( ofile ) {
(void) unlink( ofile );
if( freopen( ofile, "w", stdout ) != stdout ) {
fprintf(stderr, "%s: cannot create '%s'\n", me, ofile);
exit( 1 );
}
}
/* Take path list from command if anything left on it */
if( optind < argc ) {
struct stat st;
int others = 0;
while( optind < argc ) {
if( others++ ) {
putchar( '\n' );
}
path = argv[ optind++ ];
if( stat( path, &st ) ) {
fprintf(
stderr,
"%s: "
"cannot stat '%s'; "
"errno=%d (%s)"
".\n",
me,
path,
errno,
strerror( errno )
);
++nonfatal;
continue;
}
printName( path, &st, 0, 1 );
processDirectory( path, 1 );
}
} else {
/* Default to current directory */
char here[ PATH_MAX + 1 ];
struct stat st;
if( !getcwd( here, sizeof( here ) ) ) {
fprintf(
stderr,
"%s: getcwd() failed; errno=%d (%s).\n",
me,
errno,
strerror( errno )
);
exit( 1 );
}
if( stat( here, &st ) ) {
fprintf(
stderr,
"%s: "
"cannot stat '%s'; "
"errno=%d (%s)"
".\n",
me,
here,
errno,
strerror( errno )
);
} else {
printName( here, &st, 0, 1 );
processCurrentDirectory( here );
}
}
return( nonfatal ? 1 : 0 );
}
static int
processCurrentDirectory(
char *path
)
{
struct direct *d;
int lastentry;
++currentDepth;
do {
int Nnames;
struct direct **namelist;
struct stat *stp;
/* Get basenames from this directory, if we can */
Nnames = scandir( ".", &namelist, permit,
(f_sw ? Falphasort : alphasort) );
if( Nnames <= 0 ) {
#if 0
fprintf(
stderr,
"%s: cannot scan '%s'.\n",
me,
path
);
#endif /* NOPE */
++nonfatal;
break;
}
stp = xmalloc( Nnames * sizeof( *stp ) );
do {
int subdirs;
int i;
/* Stat the directory and count subdirectories */
for( subdirs = i = 0; i < Nnames; ++i ) {
d = namelist[ i ];
if( lstat( d->d_name, stp + i ) < 0 ) {
fprintf( stderr,
"%s: cannot stat '%s'\n", me,
d->d_name );
++nonfatal;
stp[i].st_mode = 0;
}
if( S_ISDIR( stp[i].st_mode ) ) {
++subdirs;
}
}
/* Say current directory and opt trailer line */
if( d_sw ) {
if( subdirs && !s_sw ) {
printf( "%s|\n", prefix );
}
} else if( Nnames && !s_sw ) {
printf( "%s|\n", prefix );
}
/* Process each name in the directory in order */
for( i = 0; i < Nnames; ++i ) {
struct stat * const st = stp + i;
mode_t const mode = st->st_mode;
int const isDir = S_ISDIR( mode );
d = namelist[i];
/* Show only directories if -d switch */
lastentry = ( (i+1) == Nnames ) ? 1 : 0;
if( isDir ) {
if( d_sw ) {
lastentry = ((--subdirs) <= 0) ? 1 : 0;
}
printName( d->d_name, st, lastentry, 0 );
if( currentDepth < depth ) {
/* Push new directory state */
if( lastentry ) {
strcpy( prefix + Nprefix, " " );
} else {
strcpy( prefix + Nprefix, "| " );
}
Nprefix += 4;
processDirectory( d->d_name, lastentry );
Nprefix -= 4;
}
prefix[ Nprefix ] = '\0';
if( !lastentry && !s_sw ) {
printf( "%s|\n", prefix );
}
} else if( !d_sw ) {
processFile( d->d_name, st, lastentry );
if( !lastentry && !s_sw ) {
printf( "%s|\n", prefix );
}
}
}
} while( 0 );
free( stp );
/* Don't need namelist any longer */
free( namelist );
} while( 0 );
--currentDepth;
return( 0 );
}
void SofaConfiguration::processDirectory(const QString &dir)
{
QDir d(QString(path.c_str())+dir);
d.setFilter( QDir::Dirs | QDir::Hidden | QDir::NoSymLinks );
std::vector< QString > subDir;
const QFileInfoList &listDirectories =
#ifdef SOFA_QT4
d.entryInfoList();
QStringList filters; filters << "*.h" << "*.hpp" << "*.cpp" << "*.inl"<< "*.c" << "*.cu" << "*.cuh" << "*.pro" ;
d.setNameFilters(filters);
for (int j = 0; j < listDirectories.size(); ++j)
{
QFileInfo fileInfo=listDirectories.at(j);
#else
*(d.entryInfoList());
QString filters="*.h *.hpp *.cpp *.inl *.c *.cu *.cuh *.pro";
d.setNameFilter(filters);
QFileInfoListIterator itDir( listDirectories );
while ( (itDir.current()) != 0 )
{
QFileInfo fileInfo=*(itDir.current());
#endif
subDir.push_back(fileInfo.fileName());
#ifndef SOFA_QT4
++itDir;
#endif
}
d.setFilter( QDir::Files | QDir::Hidden | QDir::NoSymLinks );
std::vector< QString > filesInside;
const QFileInfoList &listFiles =
#ifdef SOFA_QT4
d.entryInfoList();
for (int j = 0; j < listFiles.size(); ++j)
{
QFileInfo fileInfo=listFiles.at(j);
#else
*(d.entryInfoList());
QFileInfoListIterator itFile( listFiles );
while ( (itFile.current()) != 0 )
{
QFileInfo fileInfo=*(itFile.current());
#endif
filesInside.push_back(fileInfo.fileName());
processFile(fileInfo);
#ifndef SOFA_QT4
++itFile;
#endif
}
for (unsigned int i=0; i<subDir.size(); ++i)
{
if (subDir[i].left(1) == QString(".")) continue;
if (subDir[i] == QString("OBJ")) continue;
QString nextDir=dir+QString("/")+subDir[i];
processDirectory(nextDir);
}
}
void SofaConfiguration::processFile(const QFileInfo &info)
{
std::fstream file;
file.open(info.absFilePath(), std::ios::in | std::ios::out);
std::string line;
while (std::getline(file, line))
{
std::set< QWidget *>::iterator it;
for (it=optionsModified.begin(); it!=optionsModified.end(); it++)
{
std::string option((*it)->name());
if (line.find(option.c_str()) != std::string::npos)
{
//Touch the file
file.seekg(0);
char space; file.get(space);
file.seekg(0);
file.put(space);
std::cout << " found in " << info.absFilePath().ascii() << std::endl;
return;
}
}
}
file.close();
}
}
void SofaConfiguration::saveConfiguration()
{
std::ofstream out((path + std::string("/sofa-local.cfg")).c_str());
std::string currentCategory;
std::vector< CONDITION > currentConditions;
for (unsigned int i=0; i<options.size(); ++i)
{
DEFINES &option=options[i]->option;
bool differentConditions=false;
if (currentConditions.size() != option.conditions.size()) differentConditions=true;
else
{
for (unsigned int c=0; c<currentConditions.size() && !differentConditions; ++c)
{
if (currentConditions[c] != option.conditions[c]) differentConditions=true;
}
}
if (differentConditions)
{
for (unsigned int c=0; c<currentConditions.size(); ++c) out << "}\n";
currentConditions = option.conditions;
}
if (currentCategory != option.category)
{
if (!currentCategory.empty()) out << "\n\n";
currentCategory = option.category;
out << "########################################################################\n";
out << "# " << currentCategory << "\n";
out << "########################################################################\n";
}
if (differentConditions)
{
for (unsigned int c=0; c<currentConditions.size(); ++c)
{
if (!currentConditions[c].presence) out << "!";
switch( currentConditions[c].type)
{
case OPTION:
out << "contains(DEFINES," << currentConditions[c].option << "){\n";
break;
case ARCHI:
out << currentConditions[c].option << "{\n";
break;
}
}
}
if (option.typeOption)
{
std::string description=option.description;
for (unsigned int position=0; position<description.size(); ++position)
{
if (description[position] == '\n') description.insert(position+1, "# ");
}
out << "\n# Uncomment " << description << "\n";
if (!option.value) out << "# ";
out << "DEFINES += " << option.name << "\n";
}
else
{
if (!option.value) out << "# ";
out << option.name << " " << option.description << "\n";
}
}
for (unsigned int c=0; c<currentConditions.size(); ++c) out << "}\n";
out.close();
std::set< QWidget *>::iterator it;
if (!optionsModified.empty())
{
std::vector<QString> listDir;
listDir.push_back(QString("/applications"));
listDir.push_back(QString("/modules"));
listDir.push_back(QString("/framework"));
//listDir.push_back(QString("/extlibs"));
std::set< QWidget *>::iterator it;
std::cout << "Touch file containing option :";
for (it=optionsModified.begin(); it!=optionsModified.end(); it++)
std::cout << "\"" << (*it)->name() << "\" ";
std::cout << "in [ ";
for (unsigned int i=0; i<listDir.size(); ++i)
std::cout << listDir[i].ascii() << " ";
std::cout << "]" << std::endl;
for (unsigned int i=0; i<listDir.size(); ++i)
{
std::cout << " Searching in " << listDir[i].ascii() << "\n";
processDirectory(listDir[i]);
}
}
QStringList argv;
#ifdef WIN32
argv << QString("cmd.exe");
argv << QString("/c");
argv << QString(QString(projectVC->text()) );
#elif defined (__APPLE__)
argv << QString("sh");
argv << QString("Project MacOS.sh");
#else
#if SOFA_QT4
argv << QString("qmake-qt4");
#else
argv << QString("qmake");
#endif
#endif
p = new Q3Process(argv,this);
p->setWorkingDirectory(QDir(QString(path.c_str())));
connect( p, SIGNAL( readyReadStdout() ), this, SLOT( redirectStdOut() ) );
connect( p, SIGNAL( readyReadStderr() ), this, SLOT( redirectStdErr() ) );
connect( p, SIGNAL( processExited() ), this, SLOT( saveConfigurationDone() ) );
p->start();
this->saveButton->setEnabled(false);
}
int main(int argc, char *argv[]) {
LinkedList *ll = NULL;
TreeSet *ts = NULL;
char *sp;
char pattern[4096];
RegExp *reg;
Iterator *it;
if (argc < 2) {
fprintf(stderr, "Usage: ./fileCrawler pattern [dir] ...\n");
return -1;
}
/*
* convert bash expression to regular expression and compile
*/
cvtPattern(pattern, argv[1]);
if ((reg = re_create()) == NULL) {
fprintf(stderr, "Error creating Regular Expression Instance\n");
return -1;
}
if (! re_compile(reg, pattern)) {
char eb[4096];
re_status(reg, eb, sizeof eb);
fprintf(stderr, "Compile error - pattern: `%s', error message: `%s'\n",
pattern, eb);
re_destroy(reg);
return -1;
}
/*
* create linked list and treeset
*/
if ((ll = ll_create()) == NULL) {
fprintf(stderr, "Unable to create linked list\n");
goto done;
}
if ((ts = ts_create(scmp)) == NULL) {
fprintf(stderr, "Unable to create tree set\n");
goto done;
}
/*
* populate linked list
*/
if (argc == 2) {
if (! processDirectory(".", ll, 1))
goto done;
} else {
int i;
for (i = 2; i < argc; i++) {
if (! processDirectory(argv[i], ll, 1))
goto done;
}
}
/*
* for each directory in the linked list, apply regular expression
*/
while (ll_removeFirst(ll, (void **)&sp)) {
int stat = applyRe(sp, reg, ts);
free(sp);
if (! stat)
break;
}
/*
* create iterator to traverse files matching pattern in sorted order
*/
if ((it = ts_it_create(ts)) == NULL) {
fprintf(stderr, "Unable to create iterator over tree set\n");
goto done;
}
while (it_hasNext(it)) {
char *s;
(void) it_next(it, (void **)&s);
printf("%s\n", s);
}
it_destroy(it);
/*
* cleanup after ourselves so there are no memory leaks
*/
done:
if (ll != NULL)
ll_destroy(ll, free);
if (ts != NULL)
ts_destroy(ts, free);
re_destroy(reg);
return 0;
}
void processDirectory(const QString &dir_old,
const QString &dir_new,
const QString &dir_result,
const QStringList &ignoreMasks)
{
QDir qdir(dir_new, "", QDir::Name | QDir::IgnoreCase,
QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot);
QFileInfoList files = qdir.entryInfoList();
for (int i = 0; i < files.count(); ++i)
{
//qDebug() << files.at(i).absoluteFilePath();
QFileInfo fi = files.at(i);
// check for ignores
bool ignore_this_file = false;
for (int k = 0; k < ignoreMasks.size(); ++k)
{
QRegExp rx(ignoreMasks.at(k));
rx.setPatternSyntax(QRegExp::Wildcard);
if (rx.exactMatch(fi.fileName()))
{
ignore_this_file = true;
break;
}
}
if (ignore_this_file)
continue;
if (fi.isFile())
{
QDir qdir_old(dir_old);
if(!qdir_old.exists(fi.fileName()))
{
CopyFile(dir_result, fi.absoluteFilePath());
}
else // compare crc, if changed then copy
{
quint16 crc_new = getCrc(fi.absoluteFilePath());
quint16 crc_old = getCrc(qdir_old.absoluteFilePath(fi.fileName()));
log(QString("Comparing crc %1 = %2\n").arg(crc_old).arg(crc_new));
if (crc_new == 0 || crc_old == 0)
return;
if (crc_new != crc_old)
{
CopyFile(dir_result, fi.absoluteFilePath());
}
}
}
else if (fi.isDir())
{
processDirectory(dir_old + "/" + fi.fileName(),
dir_new + "/" + fi.fileName(),
dir_result + "/" + fi.fileName(),
ignoreMasks);
//*/
}
}
}
int processDirectory(char* path) {
DIR* dir;
struct dirent* direntry;
dir = opendir(path);
if( dir == NULL ) {
fprintf(stderr, "failed to open directory %s\n", path);
return 0;
}
int found = 0;
while( (direntry = readdir(dir)) ) {
if( !strcmp(direntry->d_name, "..") || !strcmp(direntry->d_name, ".") )
continue;
if( direntry->d_type == DT_DIR ) {
char* sub = malloc(strlen(path)+2+strlen(direntry->d_name));
sub[0] = 0;
strcat(sub, path);
strcat(sub, "/");
strcat(sub, direntry->d_name);
int c = checkName(direntry->d_name);
if( c == strlen(direntry->d_name) ) {
found += processDirectory(sub);
} else {
found++;
if( renameFiles ) {
printf("subdirectory contains errors, renaming: %s\n", sub);
if( verbose ) {
printHex(sub,-1);
printf("\n");
}
char* newname = malloc(strlen(direntry->d_name));
strcpy(newname,direntry->d_name);
toValidName(newname);
if( verbose )
printf("renaming this dir to %s\n", newname);
char* newsub = malloc(strlen(path)+2+strlen(newname));
newsub[0] = 0;
strcat(newsub, path);
strcat(newsub, "/");
strcat(newsub, newname);
free(newname);
if( rename(sub, newsub) )
printf("failed to rename directory %s\n", sub);
else {
found += processDirectory(newsub);
}
free(newsub);
} else {
if( deleteFiles ) //don't delete invalid directories, data inside may still be of use ;)
printf("subdirectory contains errors, skipping but won't delete: %s\n", sub);
else
printf("subdirectory contains errors, skipping: %s\n", sub);
if( verbose ) {
printHex(sub,-1);
printf("\n");
}
}
}
free(sub);
} else {
int i = checkName(direntry->d_name);
if( i != strlen(direntry->d_name) ) {
printf("invalid: %s/%s", path, direntry->d_name);
if( verbose ) {
printf(": has invalid unicode at %d, hex below:\n", i);
printHex(direntry->d_name,-1);
}
printf("\n");
char* sub = 0;
if( renameFiles || deleteFiles ) { //build path for both delete and rename functions
sub = malloc(strlen(path)+2+strlen(direntry->d_name));
sub[0] = 0;
strcat(sub, path);
strcat(sub, "/");
strcat(sub, direntry->d_name);
}
if( renameFiles ) {
char* newname = malloc(strlen(direntry->d_name)+1);
strcpy(newname,direntry->d_name);
toValidName(newname);
if( verbose )
printf("renaming this file to %s\n", newname);
char* newsub = malloc(strlen(path)+2+strlen(newname));
newsub[0] = 0;
strcat(newsub, path);
strcat(newsub, "/");
strcat(newsub, newname);
free(newname);
if( rename(sub, newsub) )
printf("failed to rename file %s\n", sub);
free(newsub);
}
if( deleteFiles ) {
if( remove(sub) )
printf("removed %s\n", sub);
else
printf("failed to remove %s\n", sub);
}
if( renameFiles || deleteFiles )
free(sub);
found++;
}
}
}
closedir(dir);
return found;
}
void LoadFileListTask::run()
{
m_cache = readCacheFile();
foreach (QString path, m_directories) {
processDirectory(path);
}
NABoolean HHDFSTableStats::populate(struct hive_tbl_desc *htd)
{
// here is the basic outline how this works:
//
// 1. Walk SD descriptors of the table, one for the table
// itself and one for each partition. Each one represents
// one HDFS directory with files for the table.
// 2. For each list partition directory (or the directory for
// an unpartitioned table):
// 3. Walk through every file. For every file:
// 4. Determine bucket number (0 if file is not bucketed)
// 5. Add file to its bucket
// 6. Walk through blocks of file. For every block:
// 7. Get host list for this block and add it
// 9. Get file stats
// 10. Aggregate file stats for all files and buckets
// 11. Aggregate bucket stats for all buckets of the partition
// 12. Aggregate partition stats for all partitions of the table
struct hive_sd_desc *hsd = htd->getSDs();
if (hsd == NULL)
return TRUE; // nothing need to be done
diags_.reset();
tableDir_ = hsd->location_;
numOfPartCols_ = htd->getNumOfPartCols();
recordTerminator_ = hsd->getRecordTerminator();
fieldTerminator_ = hsd->getFieldTerminator() ;
nullFormat_ = hsd->getNullFormat();
NAString hdfsHost;
Int32 hdfsPort = -1;
NAString tableDir;
if (hsd) {
if (hsd->isTextFile())
type_ = TEXT_;
else if (hsd->isSequenceFile())
type_ = SEQUENCE_;
else if (hsd->isOrcFile())
type_ = ORC_;
else
type_ = UNKNOWN_;
}
// split table URL into host, port and filename
if (! splitLocation(hsd->location_,
hdfsHost,
hdfsPort,
tableDir,
diags_,
hdfsPortOverride_))
return FALSE;
if (! connectHDFS(hdfsHost, hdfsPort))
return FALSE; // diags_ is set
// put back fully qualified URI
tableDir = hsd->location_;
computeModificationTSmsec();
if (diags_.isSuccess()) {
modificationTSInMillisec_ = htd->setRedeftime(modificationTSInMillisec_);
while (hsd && diags_.isSuccess()) {
// visit the directory
processDirectory(hsd->location_, hsd->buckets_,
hsd->isTrulyText(),
hsd->getRecordTerminator());
hsd = hsd->next_;
}
}
disconnectHDFS();
validationJTimestamp_ = JULIANTIMESTAMP();
return diags_.isSuccess();
}
