Exemple #1
0
static bool mapToFile(MaConn *conn, bool *rescan)
{
    MaRequest   *req;
    MaResponse  *resp;

    req = conn->request;
    resp = conn->response;

    if (resp->filename == 0) {
        resp->filename = makeFilename(conn, req->alias, req->url, 1);
    }
    req->dir = maLookupBestDir(req->host, resp->filename);

    if (req->dir == 0) {
        maFailRequest(conn, MPR_HTTP_CODE_NOT_FOUND, "Missing directory block for %s", resp->filename);
        return 0;
    }

    mprAssert(req->dir);

    req->auth = req->dir->auth;

    if (!resp->fileInfo.valid && mprGetFileInfo(conn, resp->filename, &resp->fileInfo) < 0) {
#if UNUSED
        if (req->method & (MA_REQ_GET | MA_REQ_POST)) {
            maFailRequest(conn, MPR_HTTP_CODE_NOT_FOUND, "Can't open document: %s", resp->filename);
            return 0;
        }
#endif
    }
    if (resp->fileInfo.isDir) {
        processDirectory(conn, rescan);
    }
    return 1;
}
Exemple #2
0
void Index::createBase()
{
	for (std::vector<std::string>::size_type i = 0; i < directories_.size(); ++i)
	{
		processDirectory(directories_.at(i), std::string());
	}
}
bool ICOImageDecoder::decodeDirectory()
{
    // Read and process directory.
    if ((m_decodedOffset < sizeOfDirectory) && !processDirectory())
        return false;

    // Read and process directory entries.
    return (m_decodedOffset >= (sizeOfDirectory + (m_dirEntries.size() * sizeOfDirEntry))) || processDirectoryEntries();
}
Exemple #4
0
static int processDirectory(char *dirname, LinkedList *ll, int verbose) {
   DIR *dd;
   struct dirent *dent;
   char *sp;
   int len, status = 1;
   char d[4096];

   /*
    * eliminate trailing slash, if there
    */
   strcpy(d, dirname);
   len = strlen(d);
   if (len > 1 && d[len-1] == '/')
      d[len-1] = '\0';
   /*
    * open the directory
    */
   if ((dd = opendir(d)) == NULL) {
      if (verbose)
         fprintf(stderr, "Error opening directory `%s'\n", d);
      return 1;
   }
   /*
    * duplicate directory name to insert into linked list
    */
   sp = strdup(d);
   if (sp == NULL) {
      fprintf(stderr, "Error adding `%s' to linked list\n", d);
      status = 0;
      goto cleanup;
   }
   if (!ll_add(ll, sp)) {
      fprintf(stderr, "Error adding `%s' to linked list\n", sp);
      free(sp);
      status = 0;
      goto cleanup;
   }
   if (len == 1 && d[0] == '/')
      d[0] = '\0';
   /*
    * read entries from the directory
    */
   while (status && (dent = readdir(dd)) != NULL) {
      if (strcmp(".", dent->d_name) == 0 || strcmp("..", dent->d_name) == 0)
         continue;
      if (dent->d_type & DT_DIR) {
         char b[4096];
         sprintf(b, "%s/%s", d, dent->d_name);
	 status = processDirectory(b, ll, 0);
      }
   }
cleanup:
   (void) closedir(dd);
   return status;
}
Exemple #5
0
NABoolean HHDFSTableStats::populate(struct hive_tbl_desc *htd)
{
  // here is the basic outline how this works:
  //
  // 1. Walk SD descriptors of the table, one for the table
  //    itself and one for each partition. Each one represents
  //    one HDFS directory with files for the table.
  // 2. For each list partition directory (or the directory for
  //    an unpartitioned table):
  //     3. Walk through every file. For every file:
  //         4. Determine bucket number (0 if file is not bucketed)
  //         5. Add file to its bucket
  //         6. Walk through blocks of file. For every block:
  //             7. Get host list for this block and add it
  //         9. Get file stats
  //     10. Aggregate file stats for all files and buckets
  // 11. Aggregate bucket stats for all buckets of the partition
  // 12. Aggregate partition stats for all partitions of the table

  NABoolean result = TRUE;
  struct hive_sd_desc *hsd = htd->getSDs();
  tableDir_ = hsd->location_;
  numOfPartCols_ = htd->getNumOfPartCols();
  recordTerminator_ = hsd->getRecordTerminator();
  fieldTerminator_ = hsd->getFieldTerminator() ;
  NAString hdfsHost;
  Int32 hdfsPort = -1;
  NAString tableDir;

  while (hsd)
    {
      // split table URL into host, port and filename
      splitLocation(hsd->location_, hdfsHost, hdfsPort, tableDir);
      if (! connectHDFS(hdfsHost, hdfsPort))
        CMPASSERT(fs_);

      // put back fully qualified URI
      tableDir = hsd->location_;

      // visit the directory
      result = processDirectory(tableDir, hsd->buckets_, 
                                hsd->isTrulyText(), 
                                hsd->getRecordTerminator(),
                                hsd->isSequenceFile());

      hsd = hsd->next_;
    }

  disconnectHDFS();
  validationJTimestamp_ = JULIANTIMESTAMP();

  return result;
}
int main(int argc, char** argv) {
  if( argc > 4 )
    usage();
  argc--;
  argv++;

  char* path = 0;
  while( argc ) {
    if( !strcmp(*argv, "-d") ) {
      deleteFiles = true;
      printf("will delete invalid files!\n");
    } else {
      if( !strcmp(*argv, "-v") ) {
        verbose = true;
        printf("will be verbose\n");
      } else {
        if( !strcmp(*argv, "-r") ) {
          renameFiles = true;
          printf("will rename invalid files!\n");
        } else {
          if( path != 0 ) {
            free(path);
            usage();
          } else {
            path = malloc(strlen(*argv));
            strcpy(path, *argv);
          }
        }
      }
    }
    argc--;
    argv++;
  }

  if( deleteFiles && renameFiles ) {
    printf("deleting and renaming is not possible at once\n");
    usage();
  }

  if( path == 0 ) {
    path = malloc(2);
    strcpy(path, ".");
  }

  int found = processDirectory(path);

  printf("found %d invalid files and directories\n", found);

  free(path);
  return 0;
}
void LoadFileListTask::processDirectory(const QString &path)
{
	emit currentPathChanged(path);
    QFileInfoList fileInfoList = QDir(path).entryInfoList(QDir::AllEntries | QDir::NoDotAndDotDot);
	for (int j = 0; j < fileInfoList.size(); j++) {
		QFileInfo fileInfo = fileInfoList.at(j);
		if (fileInfo.isDir()) {
			processDirectory(fileInfo.filePath());
		}
		else {
			processFile(fileInfo.filePath());
		}
    }
}
Exemple #8
0
static void processDirectory( void )
{
    DIR                 *dirh;
    struct dirent       *dp;
    dirstack            *tmp;

    if( RecurLevels != 0 ) {
        extendPath( PathBuff, "*.*" );
        dirh = opendir( PathBuff );
        if( dirh != NULL ) {
            --RecurLevels;
            for( ;; ) {
                if( DoneFlag ) return;
                dp = readdir( dirh );
                if( dp == NULL ) break;
#if defined( __WATCOMC__ ) && !defined( __UNIX__ )
                if( !( dp->d_attr & _A_SUBDIR ) ) continue;
#else
                {
                    struct stat sblk;
                    char tmp_path[_MAX_PATH+1];
                    strcpy( tmp_path, PathBuff );
                    extendPath( tmp_path, dp->d_name );
                    if( stat( tmp_path, &sblk ) == 0 && !S_ISDIR( sblk.st_mode ) ) {
                        continue;
                    }
                }
#endif
                if( dp->d_name[0] == '.' ) {
                    if( dp->d_name[1] == '.' || dp->d_name[1] == '\0' ) continue;
                }
                if( DoneFlag ) return;
                tmp = (dirstack *) SafeMalloc( sizeof( dirstack ) );
                extendPath( tmp->name, dp->d_name );
                tmp->prev = Stack;
                Stack = tmp;
                processDirectory();
                Stack = tmp->prev;
                free( tmp );
            }
            ++RecurLevels;
            closedir( dirh );
        }
    }
    executeWgrep();
}
Exemple #9
0
static void nextWgrep( char **paths )
{
    struct stat     sblk;

    Stack = (dirstack *) SafeMalloc( sizeof( dirstack ) );

    while( *paths != NULL ) {
        if( stat( *paths, &sblk ) == 0 && S_ISDIR( sblk.st_mode ) ) {
            strcpy( Stack->name, *paths );
            strcpy( CurrPattern, "*.*" );
        } else {
            splitPath( *paths, Stack->name, CurrPattern );
        }
        processDirectory( );
        paths++;
    }

    free( Stack );
}
Exemple #10
0
void Index::processDirectory(const lookupDirectory &baseDirectory, const std::string &relative)
{
	std::string directory = baseDirectory.path + relative;
	std::vector<file> content = directoryEntries(directory);

	if (baseDirectory.withSubdirectories)
	{
		for (std::vector<file>::size_type i = 0; i < content.size(); ++i)
		{
			if (content.at(i).type == FileType::Directory)
			{
				processDirectory(baseDirectory, relative + content.at(i).name + "/");
			}
		}
	}

	processDesktopInDirectory(baseDirectory.path, relative, content);
	processMimeApps(directory + "mimeapps.list");
}
/**
	Parses all the files within the given directory (and within its subdirectories) and adds them to the database.
	@param path: path to the directory to process
	@param name: name of the directory to process
	@param parseBody: boolean value stating whether the body must be parsed or not

*/
void DatasetGenerator::parseDirectory(string &path, const string &directory, bool parseBody)
{
	aParseBody=parseBody;
	//Open files where the times will be saved
	string fileFullNameParsing="./Results/Parsing/parsingTimes_WB.txt";
	string fileFullNameProcessing="./Results/Parsing/processingTimes_WB.txt";
	string fileFullNameDirs="./Results/Parsing/dirNames_WB.txt";
	parsingTimesFile.open(fileFullNameParsing.c_str(), ios::app);
	processingTimesFile.open(fileFullNameProcessing.c_str(), ios::app);
	dirNames.open(fileFullNameDirs.c_str(), ios::app);
	//Parse directory
	DB.connect();
	processDirectory(path,directory);
	DB.close();
	//close files
	parsingTimesFile.close();
	processingTimesFile.close();
	dirNames.close();	

}
/**
	Processes the given entity in order to know whether it is a directory or a file, an then call the corresponding method to process the entity.
	@param path: path to the entity to process
	@param name: name of the entity to process
*/
void DatasetGenerator::processEntity(string &path, struct dirent* entity)
{
	//PROCESS DIRECTORIES
	if(entity->d_type==DT_DIR)
	{
		//Do not process directory . and ..
		if(entity->d_name[0]=='.'){
			return;
		}
		processDirectory(path, string(entity->d_name));	
		return;
	}
	//PROCESS REGULAR FILES
	if(entity->d_type==DT_REG)
	{
		processFile(path, string(entity->d_name));
		return;
	}
	//PROCESS REST OF ENTITIES
	cout <<"Not a file or directory: "<<entity->d_name<<endl;
}
Exemple #13
0
int main(int argc, char** argv) {
    int c;
    while ((c = getopt(argc, argv, "vw")) != -1) {
        switch (c) {
            case 'v':
                verbose = 1;
            break;
            case 'w':
                sleepy = 1;
            break;
            default:
                printf("Bad arguments\n");
                printHelp();
                return 1;
        }
    }

    processExtensions();
    /*if (exts) {
        int i;
        for (i = 0; *(exts + i); i++) {
            printf("%s\n", *(exts + i));
        }
    }*/
    processDirectory(argv);

    if (exts) {
        int i;
        for (i = 0; *(exts + i); i++) {
            free(*(exts + i));
        }
    }
    free(exts);

    return 0;
}
Exemple #14
0
int main(int argc, char *argv[])
{
  int app_result = 0;

  QCoreApplication a(argc, argv);
  QCoreApplication::setApplicationName("PatchMaker");
  QCoreApplication::setApplicationVersion("1.0");

  QStringList positionalArgs;
  QStringList ignoreMasks;
  if (commandLineArgumentsValid(&positionalArgs, &ignoreMasks))
  {
    processDirectory(positionalArgs.at(0),
                     positionalArgs.at(1),
                     positionalArgs.at(2),
                     ignoreMasks);
  }
  else
  {
    app_result = 1;
  }

  return app_result;
}
Exemple #15
0
int
main(
	int		argc,
	char		*argv[]
)
{
	int		c;		/* Command line character	 */
	char *		path;		/* Path being processed		 */

	me = argv[0];
	add_ignore( "." );
	add_ignore( ".." );
	while(
		(c = getopt( argc, argv, "aDc:dhFfgl:i:n:o:psuW:w" )) != EOF
	)	{
		switch( c )	{
		default:
			fprintf( stderr, "%s: no -%c yet!\n", me, c );
			/*FALLTHRU*/
		case '?':
			++nonfatal;
			break;
		case 'D':
			++debug;
			break;
		case 'a':
			++a_sw;
			break;
		case 'c':
			whatColumn = atoi( optarg );
			break;
		case 'd':
			++d_sw;
			break;
		case 'F':
			++F_sw;
			break;
		case 'f':
			++f_sw;
			break;
		case 'g':
			combo_sw |= Iwanna_group;
			break;
		case 'h':
			combo_sw |= Iwanna_size;
			break;
		case 'i':
			add_ignore( optarg );
			break;
		case 'l':
			depth = atoi( optarg );
			break;
		case 'p':
			combo_sw |= Iwanna_perms;
			break;
		case 'o':
			ofile = optarg;
			break;
		case 's':
			++s_sw;
			break;
		case 'u':
			combo_sw |= Iwanna_user;
			break;
		case 'W':
			leadin = optarg;
			Nleadin = strlen( leadin );
			break;
		case 'w':
			++w_sw;
			break;
		}
	}
	if( nonfatal )	{
		fprintf( stderr, "%s: illegal switch(es)\n", me );
		fprintf( stderr,
			"usage: %s "
			"[-D] "
			"[-a] "
			"[-d] "
			"[-c] "
			"[-d depth] "
			"[-f] "
			"[-l limit] "
			"[-i subdir] "
			"[-s] "
			"[-o ofile] "
			"[-p] "
			"[-W prefix] "
			"[-w] "
			"[dir...]"
			"\n",
			me
		);
		exit( 1 );
	}
	if( ofile )	{
		(void) unlink( ofile );
		if( freopen( ofile, "w", stdout ) != stdout )	{
			fprintf(stderr, "%s: cannot create '%s'\n", me, ofile);
			exit( 1 );
		}
	}
	/* Take path list from command if anything left on it		 */
	if( optind < argc )	{
		struct stat	st;
		int		others = 0;

		while( optind < argc )	{
			if( others++ )	{
				putchar( '\n' );
			}
			path = argv[ optind++ ];
			if( stat( path, &st ) )	{
				fprintf(
					stderr,
					"%s: "
					"cannot stat '%s'; "
					"errno=%d (%s)"
					".\n",
					me,
					path,
					errno,
					strerror( errno )
				);
				++nonfatal;
				continue;
			}
			printName( path, &st, 0, 1 );
			processDirectory( path, 1 );
		}
	} else	{
		/* Default to current directory				 */

		char		here[ PATH_MAX + 1 ];
		struct stat	st;

		if( !getcwd( here, sizeof( here ) ) )	{
			fprintf(
				stderr,
				"%s: getcwd() failed; errno=%d (%s).\n",
				me,
				errno,
				strerror( errno )
			);
			exit( 1 );
		}
		if( stat( here, &st ) )	{
			fprintf(
				stderr,
				"%s: "
				"cannot stat '%s'; "
				"errno=%d (%s)"
				".\n",
				me,
				here,
				errno,
				strerror( errno )
			);
		} else	{
			printName( here, &st, 0, 1 );
			processCurrentDirectory( here );
		}
	}
	return( nonfatal ? 1 : 0 );
}
Exemple #16
0
static int
processCurrentDirectory(
	char		*path
)
{
	struct direct	*d;
	int		lastentry;

	++currentDepth;
	do	{
		int		Nnames;
		struct direct	**namelist;
		struct stat	*stp;

		/* Get basenames from this directory, if we can		 */
		Nnames = scandir( ".", &namelist, permit,
			(f_sw ? Falphasort : alphasort) );
		if( Nnames <= 0 )	{
#if	0
			fprintf(
				stderr,
				"%s: cannot scan '%s'.\n",
				me,
				path
			);
#endif	/* NOPE */
			++nonfatal;
			break;
		}
		stp = xmalloc( Nnames * sizeof( *stp ) );
		do	{
			int		subdirs;
			int		i;

			/* Stat the directory and count subdirectories	 */
			for( subdirs = i = 0; i < Nnames; ++i )	{
				d = namelist[ i ];
				if( lstat( d->d_name, stp + i ) < 0 )	{
					fprintf( stderr,
						"%s: cannot stat '%s'\n", me,
						d->d_name );
					++nonfatal;
					stp[i].st_mode = 0;
				}
				if( S_ISDIR( stp[i].st_mode ) )	{
					++subdirs;
				}
			}
			/* Say current directory and opt trailer line	 */
			if( d_sw )	{
				if( subdirs && !s_sw )	{
					printf( "%s|\n", prefix );
				}
			} else if( Nnames && !s_sw )	{
				printf( "%s|\n", prefix );
			}
			/* Process each name in the directory in order	 */
			for( i = 0; i < Nnames; ++i )	{
				struct stat * const	st = stp + i;
				mode_t const		mode = st->st_mode;
				int const		isDir = S_ISDIR( mode );

				d = namelist[i];
				/* Show only directories if -d switch	 */
				lastentry = ( (i+1) == Nnames ) ? 1 : 0;
				if( isDir )	{
					if( d_sw )	{
						lastentry = ((--subdirs) <= 0) ? 1 : 0;
					}
					printName( d->d_name, st, lastentry, 0 );
					if( currentDepth < depth )	{
						/* Push new directory state */
						if( lastentry )	{
							strcpy( prefix + Nprefix, "    " );
						} else	{
							strcpy( prefix + Nprefix, "|   " );
						}
						Nprefix += 4;
						processDirectory( d->d_name, lastentry );
						Nprefix -= 4;
					}
					prefix[ Nprefix ] = '\0';
					if( !lastentry && !s_sw )	{
						printf( "%s|\n", prefix );
					}
				} else if( !d_sw )	{
					processFile( d->d_name, st, lastentry );
					if( !lastentry && !s_sw )	{
						printf( "%s|\n", prefix );
					}
				}
			}
		} while( 0 );
		free( stp );
		/* Don't need namelist any longer			 */
		free( namelist );
	} while( 0 );
	--currentDepth;
	return( 0 );
}
Exemple #17
0
void SofaConfiguration::processDirectory(const QString &dir)
{

    QDir d(QString(path.c_str())+dir);

    d.setFilter( QDir::Dirs | QDir::Hidden | QDir::NoSymLinks );

    std::vector< QString > subDir;

    const QFileInfoList &listDirectories =
#ifdef SOFA_QT4
        d.entryInfoList();
    QStringList filters; filters << "*.h" << "*.hpp" << "*.cpp" << "*.inl"<< "*.c" << "*.cu" << "*.cuh" << "*.pro" ;
    d.setNameFilters(filters);
    for (int j = 0; j < listDirectories.size(); ++j)
    {
        QFileInfo fileInfo=listDirectories.at(j);
#else
        *(d.entryInfoList());
    QString filters="*.h *.hpp *.cpp *.inl *.c *.cu *.cuh *.pro";
    d.setNameFilter(filters);
    QFileInfoListIterator itDir( listDirectories );
    while ( (itDir.current()) != 0 )
    {

        QFileInfo fileInfo=*(itDir.current());
#endif
        subDir.push_back(fileInfo.fileName());
#ifndef SOFA_QT4
        ++itDir;
#endif
    }

    d.setFilter( QDir::Files | QDir::Hidden | QDir::NoSymLinks );


    std::vector< QString > filesInside;


    const QFileInfoList &listFiles =
#ifdef SOFA_QT4
        d.entryInfoList();
    for (int j = 0; j < listFiles.size(); ++j)
    {
        QFileInfo fileInfo=listFiles.at(j);
#else
        *(d.entryInfoList());
    QFileInfoListIterator itFile( listFiles );
    while ( (itFile.current()) != 0 )
    {
        QFileInfo fileInfo=*(itFile.current());
#endif
        filesInside.push_back(fileInfo.fileName());
        processFile(fileInfo);
#ifndef SOFA_QT4
        ++itFile;
#endif
    }

    for (unsigned int i=0; i<subDir.size(); ++i)
    {
        if (subDir[i].left(1) == QString(".")) continue;
        if (subDir[i] == QString("OBJ"))       continue;

        QString nextDir=dir+QString("/")+subDir[i];
        processDirectory(nextDir);
    }
}

void SofaConfiguration::processFile(const QFileInfo &info)
{
    std::fstream file;
    file.open(info.absFilePath(), std::ios::in | std::ios::out);
    std::string line;
    while (std::getline(file, line))
    {
        std::set< QWidget *>::iterator it;
        for (it=optionsModified.begin(); it!=optionsModified.end(); it++)
        {
            std::string option((*it)->name());
            if (line.find(option.c_str()) != std::string::npos)
            {
                //Touch the file
                file.seekg(0);
                char space; file.get(space);
                file.seekg(0);
                file.put(space);
                std::cout << "      found in " << info.absFilePath().ascii() << std::endl;
                return;
            }
        }
    }
    file.close();
}
}
Exemple #18
0
void SofaConfiguration::saveConfiguration()
{
    std::ofstream out((path + std::string("/sofa-local.cfg")).c_str());
    std::string currentCategory;
    std::vector< CONDITION > currentConditions;
    for (unsigned int i=0; i<options.size(); ++i)
    {
        DEFINES &option=options[i]->option;


        bool differentConditions=false;
        if (currentConditions.size() != option.conditions.size()) differentConditions=true;
        else
        {
            for (unsigned int c=0; c<currentConditions.size() && !differentConditions; ++c)
            {
                if (currentConditions[c] != option.conditions[c]) differentConditions=true;
            }
        }

        if (differentConditions)
        {
            for (unsigned int c=0; c<currentConditions.size(); ++c) out << "}\n";

            currentConditions = option.conditions;
        }

        if (currentCategory != option.category)
        {
            if (!currentCategory.empty())  out << "\n\n";
            currentCategory = option.category;
            out << "########################################################################\n";
            out << "# " << currentCategory << "\n";
            out << "########################################################################\n";
        }

        if (differentConditions)
        {
            for (unsigned int c=0; c<currentConditions.size(); ++c)
            {
                if (!currentConditions[c].presence) out << "!";
                switch( currentConditions[c].type)
                {
                case OPTION:
                    out << "contains(DEFINES," << currentConditions[c].option << "){\n";
                    break;
                case ARCHI:
                    out << currentConditions[c].option << "{\n";
                    break;
                }
            }
        }


        if (option.typeOption)
        {
            std::string description=option.description;
            for (unsigned int position=0; position<description.size(); ++position)
            {
                if (description[position] == '\n') description.insert(position+1, "# ");
            }
            out << "\n# Uncomment " << description << "\n";
            if (!option.value) out << "# ";
            out << "DEFINES += " << option.name << "\n";
        }
        else
        {
            if (!option.value) out << "# ";
            out << option.name << " " << option.description << "\n";
        }
    }

    for (unsigned int c=0; c<currentConditions.size(); ++c) out << "}\n";

    out.close();


    std::set< QWidget *>::iterator it;
    if (!optionsModified.empty())
    {
        std::vector<QString> listDir;
        listDir.push_back(QString("/applications"));
        listDir.push_back(QString("/modules"));
        listDir.push_back(QString("/framework"));
        //listDir.push_back(QString("/extlibs"));

        std::set< QWidget *>::iterator it;
        std::cout << "Touch file containing option :";
        for (it=optionsModified.begin(); it!=optionsModified.end(); it++)
            std::cout << "\"" << (*it)->name() << "\" ";
        std::cout << "in [ ";
        for (unsigned int i=0; i<listDir.size(); ++i)
            std::cout << listDir[i].ascii() << " ";
        std::cout << "]" << std::endl;


        for (unsigned int i=0; i<listDir.size(); ++i)
        {
            std::cout << "   Searching in " << listDir[i].ascii() << "\n";
            processDirectory(listDir[i]);
        }

    }
    QStringList argv;
#ifdef WIN32
    argv << QString("cmd.exe");
    argv << QString("/c");
    argv << QString(QString(projectVC->text()) );
#elif defined (__APPLE__)
    argv << QString("sh");
    argv << QString("Project MacOS.sh");
#else
#if SOFA_QT4
    argv << QString("qmake-qt4");
#else
    argv << QString("qmake");
#endif
#endif
    p = new Q3Process(argv,this);
    p->setWorkingDirectory(QDir(QString(path.c_str())));
    connect( p, SIGNAL( readyReadStdout() ), this, SLOT( redirectStdOut() ) );
    connect( p, SIGNAL( readyReadStderr() ), this, SLOT( redirectStdErr() ) );
    connect( p, SIGNAL( processExited() ), this, SLOT( saveConfigurationDone() ) );
    p->start();

    this->saveButton->setEnabled(false);
}
Exemple #19
0
int main(int argc, char *argv[]) {
   LinkedList *ll = NULL;
   TreeSet *ts = NULL;
   char *sp;
   char pattern[4096];
   RegExp *reg;
   Iterator *it;

   if (argc < 2) {
      fprintf(stderr, "Usage: ./fileCrawler pattern [dir] ...\n");
      return -1;
   }
   /*
    * convert bash expression to regular expression and compile
    */
   cvtPattern(pattern, argv[1]);
   if ((reg = re_create()) == NULL) {
      fprintf(stderr, "Error creating Regular Expression Instance\n");
      return -1;
   }
   if (! re_compile(reg, pattern)) {
      char eb[4096];
      re_status(reg, eb, sizeof eb);
      fprintf(stderr, "Compile error - pattern: `%s', error message: `%s'\n",
              pattern, eb);
      re_destroy(reg);
      return -1;
   }
   /*
    * create linked list and treeset
    */
   if ((ll = ll_create()) == NULL) {
      fprintf(stderr, "Unable to create linked list\n");
      goto done;
   }
   if ((ts = ts_create(scmp)) == NULL) {
      fprintf(stderr, "Unable to create tree set\n");
      goto done;
   }
   /*
    * populate linked list
    */
   if (argc == 2) {
      if (! processDirectory(".", ll, 1))
         goto done;
   } else {
      int i;
      for (i = 2; i < argc; i++) {
         if (! processDirectory(argv[i], ll, 1))
            goto done;
      }
   }
   /*
    * for each directory in the linked list, apply regular expression
    */
   while (ll_removeFirst(ll, (void **)&sp)) {
      int stat = applyRe(sp, reg, ts);
      free(sp);
      if (! stat)
         break;
   }
   /*
    * create iterator to traverse files matching pattern in sorted order
    */
   if ((it = ts_it_create(ts)) == NULL) {
      fprintf(stderr, "Unable to create iterator over tree set\n");
      goto done;
   }
   while (it_hasNext(it)) {
      char *s;
      (void) it_next(it, (void **)&s);
      printf("%s\n", s);
   }
   it_destroy(it);
/*
 * cleanup after ourselves so there are no memory leaks
 */
done:
   if (ll != NULL)
      ll_destroy(ll, free);
   if (ts != NULL)
      ts_destroy(ts, free);
   re_destroy(reg);
   return 0;
}
Exemple #20
0
void processDirectory(const QString &dir_old,
                      const QString &dir_new,
                      const QString &dir_result,
                      const QStringList &ignoreMasks)
{
  QDir qdir(dir_new, "", QDir::Name | QDir::IgnoreCase,
            QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot);

  QFileInfoList files = qdir.entryInfoList();
  for (int i = 0; i < files.count(); ++i)
  {
    //qDebug() << files.at(i).absoluteFilePath();
    QFileInfo fi = files.at(i);

    // check for ignores
    bool ignore_this_file = false;
    for (int k = 0; k < ignoreMasks.size(); ++k)
    {
      QRegExp rx(ignoreMasks.at(k));
      rx.setPatternSyntax(QRegExp::Wildcard);
      if (rx.exactMatch(fi.fileName()))
      {
        ignore_this_file = true;
        break;
      }
    }
    if (ignore_this_file)
      continue;

    if (fi.isFile())
    {
      QDir qdir_old(dir_old);
      if(!qdir_old.exists(fi.fileName()))
      {
        CopyFile(dir_result, fi.absoluteFilePath());
      }
      else  // compare crc, if changed then copy
      {

        quint16 crc_new = getCrc(fi.absoluteFilePath());
        quint16 crc_old = getCrc(qdir_old.absoluteFilePath(fi.fileName()));
        log(QString("Comparing crc %1 = %2\n").arg(crc_old).arg(crc_new));
        if (crc_new == 0 || crc_old == 0)
          return;

        if (crc_new != crc_old)
        {
          CopyFile(dir_result, fi.absoluteFilePath());
        }
      }
    }
    else if (fi.isDir())
    {
      processDirectory(dir_old + "/" + fi.fileName(),
                       dir_new + "/" + fi.fileName(),
                       dir_result + "/" + fi.fileName(),
                       ignoreMasks);
      //*/
    }
  }
}
int processDirectory(char* path) {
  DIR* dir;
  struct dirent* direntry;

  dir = opendir(path);
  if( dir == NULL ) {
    fprintf(stderr, "failed to open directory %s\n", path);
    return 0;
  }

  int found = 0;

  while( (direntry = readdir(dir)) ) {
    if( !strcmp(direntry->d_name, "..") || !strcmp(direntry->d_name, ".") )
      continue;
    if( direntry->d_type == DT_DIR ) {
      char* sub = malloc(strlen(path)+2+strlen(direntry->d_name));
      sub[0] = 0;
      strcat(sub, path);
      strcat(sub, "/");
      strcat(sub, direntry->d_name);
      int c = checkName(direntry->d_name);
      if( c == strlen(direntry->d_name) ) {
        found += processDirectory(sub);
      } else {
        found++;
        if( renameFiles ) {
          printf("subdirectory contains errors, renaming: %s\n", sub);
          if( verbose ) {
            printHex(sub,-1);
            printf("\n");
          }

          char* newname = malloc(strlen(direntry->d_name));
          strcpy(newname,direntry->d_name);
          toValidName(newname);
          if( verbose )
            printf("renaming this dir to %s\n", newname);

          char* newsub = malloc(strlen(path)+2+strlen(newname));
          newsub[0] = 0;
          strcat(newsub, path);
          strcat(newsub, "/");
          strcat(newsub, newname);
          free(newname);
          if( rename(sub, newsub) )
            printf("failed to rename directory %s\n", sub);
          else {
            found += processDirectory(newsub);
          }
          free(newsub);
        } else {
          if( deleteFiles ) //don't delete invalid directories, data inside may still be of use ;)
            printf("subdirectory contains errors, skipping but won't delete: %s\n", sub);
          else
            printf("subdirectory contains errors, skipping: %s\n", sub);
          if( verbose ) {
            printHex(sub,-1);
            printf("\n");
          }
        }
      }
      free(sub);
    } else {
      int i = checkName(direntry->d_name);
      if( i != strlen(direntry->d_name) ) {
        printf("invalid: %s/%s", path, direntry->d_name);
        if( verbose ) {
          printf(": has invalid unicode at %d, hex below:\n", i);
          printHex(direntry->d_name,-1);
        }
        printf("\n");

        char* sub = 0;
        if( renameFiles || deleteFiles ) { //build path for both delete and rename functions
          sub = malloc(strlen(path)+2+strlen(direntry->d_name));
          sub[0] = 0;
          strcat(sub, path);
          strcat(sub, "/");
          strcat(sub, direntry->d_name);
        }

        if( renameFiles ) {
          char* newname = malloc(strlen(direntry->d_name)+1);
          strcpy(newname,direntry->d_name);
          toValidName(newname);
          if( verbose )
            printf("renaming this file to %s\n", newname);
          char* newsub = malloc(strlen(path)+2+strlen(newname));
          newsub[0] = 0;
          strcat(newsub, path);
          strcat(newsub, "/");
          strcat(newsub, newname);
          free(newname);
          if( rename(sub, newsub) )
            printf("failed to rename file %s\n", sub);
          free(newsub);
        }

        if( deleteFiles ) {
          if( remove(sub) )
            printf("removed %s\n", sub);
          else
            printf("failed to remove %s\n", sub);
        }

        if( renameFiles || deleteFiles )
          free(sub);

        found++;
      }
    }
  }

  closedir(dir);
  return found;
}
void LoadFileListTask::run()
{
	m_cache = readCacheFile();
	foreach (QString path, m_directories) {
		processDirectory(path);
	}
NABoolean HHDFSTableStats::populate(struct hive_tbl_desc *htd)
{
  // here is the basic outline how this works:
  //
  // 1. Walk SD descriptors of the table, one for the table
  //    itself and one for each partition. Each one represents
  //    one HDFS directory with files for the table.
  // 2. For each list partition directory (or the directory for
  //    an unpartitioned table):
  //     3. Walk through every file. For every file:
  //         4. Determine bucket number (0 if file is not bucketed)
  //         5. Add file to its bucket
  //         6. Walk through blocks of file. For every block:
  //             7. Get host list for this block and add it
  //         9. Get file stats
  //     10. Aggregate file stats for all files and buckets
  // 11. Aggregate bucket stats for all buckets of the partition
  // 12. Aggregate partition stats for all partitions of the table

  struct hive_sd_desc *hsd = htd->getSDs();
  if (hsd == NULL)
    return TRUE; // nothing need to be done

  diags_.reset();
  tableDir_ = hsd->location_;
  numOfPartCols_ = htd->getNumOfPartCols();
  recordTerminator_ = hsd->getRecordTerminator();
  fieldTerminator_ = hsd->getFieldTerminator() ;
  nullFormat_ = hsd->getNullFormat();
  NAString hdfsHost;
  Int32 hdfsPort = -1;
  NAString tableDir;

  if (hsd) {
     if (hsd->isTextFile())
        type_ = TEXT_;
     else if (hsd->isSequenceFile())
        type_ = SEQUENCE_;
     else if (hsd->isOrcFile())
        type_ = ORC_;
     else
        type_ = UNKNOWN_;
  }
  // split table URL into host, port and filename
  if (! splitLocation(hsd->location_,
                hdfsHost,
                hdfsPort,
                tableDir,
                diags_,
                hdfsPortOverride_)) 
      return FALSE;
  if (! connectHDFS(hdfsHost, hdfsPort)) 
     return FALSE; // diags_ is set
  // put back fully qualified URI
  tableDir = hsd->location_;
  computeModificationTSmsec();
  if (diags_.isSuccess()) {
     modificationTSInMillisec_ = htd->setRedeftime(modificationTSInMillisec_);
     while (hsd && diags_.isSuccess()) {
        // visit the directory
        processDirectory(hsd->location_, hsd->buckets_, 
                       hsd->isTrulyText(), 
                       hsd->getRecordTerminator());

        hsd = hsd->next_;
     }
  }

  disconnectHDFS();
  validationJTimestamp_ = JULIANTIMESTAMP();

  return diags_.isSuccess();
}