void GermanStemmer::strip(StringBuffer& buffer)
{
    bool doMore = true;
    while ( doMore && buffer.length() > 3 ) {
        if ( ( buffer.length() + substCount > 5 ) &&
                buffer.substringEquals( buffer.length() - 2, buffer.length(), _T("nd"), 2 ) )
        {
            buffer.deleteChars( buffer.length() - 2, buffer.length() );
        }
        else if ( ( buffer.length() + substCount > 4 ) &&
                  buffer.substringEquals( buffer.length() - 2, buffer.length(), _T("em"), 2 ) ) {
            buffer.deleteChars( buffer.length() - 2, buffer.length() );
        }
        else if ( ( buffer.length() + substCount > 4 ) &&
                  buffer.substringEquals( buffer.length() - 2, buffer.length(), _T("er"), 2 ) ) {
            buffer.deleteChars( buffer.length() - 2, buffer.length() );
        }
        else if ( buffer.charAt( buffer.length() - 1 ) == _T('e') ) {
            buffer.deleteCharAt( buffer.length() - 1 );
        }
        else if ( buffer.charAt( buffer.length() - 1 ) == _T('s') ) {
            buffer.deleteCharAt( buffer.length() - 1 );
        }
        else if ( buffer.charAt( buffer.length() - 1 ) == _T('n') ) {
            buffer.deleteCharAt( buffer.length() - 1 );
        }
        // "t" occurs only as suffix of verbs.
        else if ( buffer.charAt( buffer.length() - 1 ) == _T('t') ) {
            buffer.deleteCharAt( buffer.length() - 1 );
        }
        else {
            doMore = false;
        }
    }
}
Beispiel #2
0
 void GermanStemmer::resubstitute(StringBuffer& buffer) {
   for ( size_t c = 0; c < buffer.length(); c++ ) {
     if ( buffer.charAt( c ) == _T('*') ) {
       TCHAR x = buffer.charAt( c - 1 );
       buffer.setCharAt( c, x );
     }
     else if ( buffer.charAt( c ) == _T('$') ) {
       buffer.setCharAt( c, 's' );
       TCHAR ch[] = { _T('c'), _T('h')};
       buffer.insert( c + 1, ch );
     }
     else if ( buffer.charAt( c ) == _T('§') ) {
       buffer.setCharAt( c, _T('c') );
       buffer.insert( c + 1, _T('h') );
     }
     else if ( buffer.charAt( c ) == _T('%') ) {
       buffer.setCharAt( c, _T('e') );
       buffer.insert( c + 1, _T('i') );
     }
     else if ( buffer.charAt( c ) == _T('&') ) {
       buffer.setCharAt( c, _T('i') );
       buffer.insert( c + 1, _T('e') );
     }
     else if ( buffer.charAt( c ) == _T('#') ) {
       buffer.setCharAt( c, _T('i') );
       buffer.insert( c + 1, _T('g') );
     }
     else if ( buffer.charAt( c ) == _T('!') ) {
       buffer.setCharAt( c, _T('s') );
       buffer.insert( c + 1, _T('t') );
     }
   }
 }
Beispiel #3
0
IClientWsWorkunits * createWorkunitsClient(IProperties * _globals)
{
    Owned<IClientWsWorkunits> wuclient = createWsWorkunitsClient();
    
    const char* eclwatch = _globals->queryProp("server");
    if(eclwatch == NULL)
        throw MakeStringException(0, "Server url not defined");
    
    StringBuffer url;
    if(Utils::strncasecmp(eclwatch, "http://", 7) != 0 && Utils::strncasecmp(eclwatch, "https://", 8) != 0)
        url.append("http://");
    url.append(eclwatch);
    if(strchr(url.str() + 7, ':') == NULL)
        url.append(":8010/");
    if(url.charAt(url.length() - 1) != '/')
        url.append("/");
    url.append("WsWorkUnits");
    wuclient->addServiceUrl(url.str());
    const char* username = _globals->queryProp("user");
    if (!username)
        username = _globals->queryProp("owner");
    const char* password = _globals->queryProp("password");
    if(username != NULL)
        wuclient->setUsernameToken(username, password, NULL);

    return LINK(wuclient.get());
}
Beispiel #4
0
StringBuffer::StringBuffer(const StringBuffer& obj) {

	
	str_->strlen = obj.length();
	str_->string_buf = new char[str_->strlen];
	for (int i = 0; i<str_->strlen; i++)
	{
		str_->string_buf[i] = obj.charAt(i);
	}
	reflink* rlinking = new reflink;
	rlinking->_address = this;
	rlinking->next = NULL;
	if (str_->_reflink == NULL)
	{
		str_->_reflink = rlinking;
	}
	else
	{
		reflink* ptr = str_->_reflink;
		reflink*  prev = NULL;
		while (ptr)
		{
			prev = ptr;
			ptr = ptr->next;
		}
		prev->next = rlinking;
	}
}
StringBuffer::StringBuffer(const StringBuffer& obj){
	
	// -> shallow copying constructor 
	_str->_length = obj.length();
	_str->_strbuf= new char[_str->_length];
	for (int i = 0;i<_str->_length;i++)
	{
		_str->_strbuf[i]=obj.charAt(i);
	}
	reflink* rlinking= new reflink;
	rlinking->_address=this;
	rlinking->next=NULL;
	if(_str->_reflink==NULL)
	{
		_str->_reflink=rlinking;
	}
	else
	{
		reflink* ptr = _str->_reflink;
		reflink*  prev = NULL;
		while(ptr)
		{
			prev= ptr;
			ptr= ptr->next;
		}
		prev->next=rlinking;
	}
}
StringBuffer::StringBuffer(const StringBuffer& object) {
	strlen = object.length(); 
	strBuffer = new char[strlen];  
	for (int i = 0; i < strlen; i++)
	{
		strBuffer[i] = object.charAt(i); 
	}
}
bool CLogContentFilter::readLogFilters(IPropertyTree* cfg, unsigned groupID)
{
    Owned<CESPLogContentGroupFilters> espLogContentGroupFilters = new CESPLogContentGroupFilters((ESPLogContentGroup) groupID);
    StringBuffer xpath;
    if (groupID != ESPLCGAll)
        xpath.appendf("Filters/Filter[@type='%s']", espLogContentGroupNames[groupID]);
    else
        xpath.append("Filters/Filter");
    Owned<IPropertyTreeIterator> filters = cfg->getElements(xpath.str());
    ForEach(*filters)
    {
        IPropertyTree &filter = filters->query();
        StringBuffer value = filter.queryProp("@value");
        if (!value.length())
            continue;

        //clean "//"
        unsigned idx = value.length()-1;
        while (idx)
        {
            if ((value.charAt(idx-1) == '/') && (value.charAt(idx) == '/'))
                value.remove(idx, 1);
            idx--;
        }

        //clean "/*" at the end
        while ((value.length() > 1) && (value.charAt(value.length()-2) == '/') && (value.charAt(value.length()-1) == '*'))
            value.setLength(value.length() - 2);

        if (value.length() && !streq(value.str(), "*") && !streq(value.str(), "/") && !streq(value.str(), "*/"))
        {
            espLogContentGroupFilters->addFilter(value.str());
        }
        else
        {
            espLogContentGroupFilters->clearFilters();
            break;
        }
    }

    bool hasFilter = espLogContentGroupFilters->getFilterCount() > 0;
    if (hasFilter)
        groupFilters.append(*espLogContentGroupFilters.getClear());
    return hasFilter;
}
void GermanStemmer::optimize(StringBuffer& buffer) {
    // Additional step for female plurals of professions and inhabitants.
    if ( buffer.length() > 5 && buffer.substringEquals( buffer.length() - 5, buffer.length(), _T("erin*"), 5 ) ) {
        buffer.deleteCharAt( buffer.length() -1 );
        strip( buffer );
    }
    // Additional step for irregular plural nouns like "Matrizen -> Matrix".
    if ( buffer.charAt( buffer.length() - 1 ) == ( _T('z') ) ) {
        buffer.setCharAt( buffer.length() - 1, _T('x') );
    }
}
void CWizardInputs::setWizardRules()
{ 
   const char* roxieRedTypes[] = {"Full", "Circular", "None", "Overloaded"};
   m_roxieAgentRedType.clear().append("Circular");
   m_roxieAgentRedChannels = 2;
   m_roxieAgentRedOffset = 1;
   m_genOptForAllComps = GENOPTIONAL_ALL;

   if(m_algProp)
   {
     Owned<IPropertyIterator> iter = m_algProp->getIterator();
     StringBuffer prop;
     ForEach(*iter)
     {
       m_algProp->getProp(iter->getPropKey(), prop.clear());
       if(prop.length() && prop.charAt(prop.length()-1) == ',')
          prop.setCharAt((prop.length()-1),' ');

       if(!strcmp(iter->getPropKey(), "max_comps_per_node"))
       {
         m_maxCompOnNode = atoi(prop.str());
       }
       else if(!strcmp(iter->getPropKey(), "avoid_combo"))
       {
         StringArray pairValue;
         DelimToStringArray(prop.str(), pairValue, ",");
         if( pairValue.ordinality() > 0)
         {
           for( unsigned i = 0; i < pairValue.ordinality() ; i++)
           {
             StringArray eachpair;
             DelimToStringArray(pairValue.item(i), eachpair, "-");
             if(eachpair.ordinality() == 2 )
             {
               StringArray* serverCompArr = 0;
               ForEachItemIn(x, eachpair)
               {
                 StringArrayPtr* pairServerArr = m_invalidServerCombo.getValue(eachpair.item(x));
                 if(pairServerArr)  
                 {
                   serverCompArr = (*pairServerArr);
                   serverCompArr->append(x == 0 ? eachpair.item(1): eachpair.item(0));
                 }
                 else
                 {
                   serverCompArr = new StringArray();
                   serverCompArr->append(x == 0 ? eachpair.item(1): eachpair.item(0));
                   m_invalidServerCombo.setValue(eachpair.item(x),serverCompArr);
                 }
               }
             }
           }
         }
       }
static inline void appendFileName(StringBuffer &path, const char *filename)
{
    if (!filename)
        return;
    while (*filename==PATHSEPCHAR)
        filename++;
    if (!*filename)
        return;
    if (path.length() && path.charAt(path.length()-1) != PATHSEPCHAR)
        path.append(PATHSEPCHAR);
    path.append(filename);
}
String AbstractContextI::loadValueFromFile(const String& fileurl) const throw (Exception)
{
	StringBuffer buf;
	buf->append(fileurl);
	if (0 != buf->indexOf(L"file://")) throw NamingException(WITHDETAILS(L"Expected url starting wtih 'file://' but found : " + fileurl));
	
	bool mustprependworkingdirectory = L'/' != buf->charAt(7);
	
	StringBuffer filename;
	if (L':' == buf->charAt(9)) // windows filepath
	{
		filename->append(buf->substring(8));
		filename = filename->replaceAll(L"/", L"\\");
	}
	else
	{
		filename->append(buf->substring(7));
	}


	String fullpath;
	if (mustprependworkingdirectory)
	{
		Anything value;
		lookup(L"/env/jlj_context_working_directory", value);
		String workingdirectory = value;
		fullpath = workingdirectory + filename;
	}
	else
	{
		fullpath = filename->toString();
	}
	if (verboseOutput()) cout << "JNDI context : Reading from file : " << fullpath << endl;
	InputStream file = new FileInputStreamI(fullpath);
	UTF8String content;
	file->read(content, 0);
	file->close();
	return content->toString();
}
void GermanStemmer::resubstitute(StringBuffer& buffer) {
    for ( size_t i = 0; i < buffer.length(); i++ ) {
#ifdef _UCS2
        TCHAR c = buffer.charAt(i);
#else
        unsigned char c = buffer.charAt(i);
#endif
        if ( c == _T('*') ) {
            buffer.setCharAt( i, buffer.charAt( i - 1 ) );
        }
        else if ( c == _T('$') ) {
            buffer.setCharAt( i, 's' );
            buffer.insert( i + 1, _T("ch"), 2 );
        }
        else if ( c == 0xa7 ) { // section sign in UTF-16
            buffer.setCharAt( i, _T('c') );
            buffer.insert( i + 1, _T('h') );
        }
        else if ( c == _T('%') ) {
            buffer.setCharAt( i, _T('e') );
            buffer.insert( i + 1, _T('i') );
        }
        else if ( c == _T('&') ) {
            buffer.setCharAt( i, _T('i') );
            buffer.insert( i + 1, _T('e') );
        }
        else if ( c == _T('#') ) {
            buffer.setCharAt( i, _T('i') );
            buffer.insert( i + 1, _T('g') );
        }
        else if ( c == _T('!') ) {
            buffer.setCharAt( i, _T('s') );
            buffer.insert( i + 1, _T('t') );
        }
    }
}
Beispiel #13
0
static void json_create_zval(Variant &z, StringBuffer &buf, int type) {
  switch (type) {
  case KindOfInt64:
    {
      const char *p = buf.data();
      ASSERT(p);
      if (p == NULL) {
        z = 0LL;
        return;
      }

      bool neg = (buf.charAt(0) == '-');

      int len = buf.size();
      if (neg) len--;
      if (len >= MAX_LENGTH_OF_LONG - 1) {
        if (len == MAX_LENGTH_OF_LONG - 1) {
          int cmp = strcmp(p + (neg ? 1 : 0), long_min_digits);
          if (!(cmp < 0 || (cmp == 0 && neg))) {
            z = strtod(p, NULL);
            return;
          }
        } else {
          z = strtod(p, NULL);
          return;
        }
      }
      z = strtoll(buf.data(), NULL, 10);
    }
    break;
  case KindOfDouble:
    z = buf.data() ? strtod(buf.data(), NULL) : 0.0;
    break;
  case KindOfString:
    z = buf.detach();
    buf.reset();
    break;
  case KindOfBoolean:
    z = (buf.data() && (*buf.data() == 't'));
    break;
  default:
    z = null;
    break;
  }
}
Beispiel #14
0
IHqlExpression * HqlCppCaseInfo::buildIndexedMap(BuildCtx & ctx, IHqlExpression * test, unsigned lower, unsigned upper)
{
    ITypeInfo * compareType = test->queryType()->queryPromotedType();
    type_t compareTypeCode = compareType->getTypeCode();

    HqlExprArray values;
    IHqlExpression * dft = queryActiveTableSelector();  // value doesn't matter as long as it will not occur
    unsigned num = (upper-lower+1);
    values.ensure(num);
    unsigned idx;
    for (idx = 0; idx < num; idx++)
        values.append(*LINK(dft));

    ForEachItemIn(idx2, pairs)
    {
        IHqlExpression & cur = pairs.item(idx2);
        IValue * value = cur.queryChild(0)->queryValue();
        unsigned replaceIndex;
        switch (compareTypeCode)
        {
        case type_int:
            replaceIndex = (int)value->getIntValue()-lower;
            break;
        case type_string:
            {
                StringBuffer temp;
                value->getStringValue(temp);
                replaceIndex = (int)(unsigned char)temp.charAt(0)-lower;
                break;
            }
        default:
            throwUnexpectedType(compareType);
        }

        IHqlExpression * mapTo = cur.queryChild(1);
        if (mapTo->getOperator() != no_constant)
            throwUnexpected();
        if (replaceIndex >= num)
            translator.reportWarning(CategoryIgnored, HQLWRN_CaseCanNeverMatch, "CASE entry %d can never match the test condition", replaceIndex);
        else
            values.replace(*LINK(mapTo),replaceIndex);
    }
Beispiel #15
0
CDfuPlusHelper::CDfuPlusHelper(IProperties* _globals,   CDfuPlusMessagerIntercept *_msgintercept)
{
    msgintercept = _msgintercept;
    globals.setown(_globals);
    sprayclient.setown(createFileSprayClient());

    const char* server = globals->queryProp("server");
    if(server == NULL)
        throw MakeStringException(-1, "Server url not specified");
    
    StringBuffer url;
    if(Utils::strncasecmp(server, "http://", 7) != 0 && Utils::strncasecmp(server, "https://", 8) != 0)
        url.append("http://");
    url.append(server);

    if(strchr(url.str() + 7, ':') == NULL)
    url.append(":8010/");

    if(url.charAt(url.length() - 1) != '/')
        url.append("/");

    StringBuffer fsurl(url.str());
    fsurl.append("FileSpray");
    sprayclient->addServiceUrl(fsurl.str());
    
    dfuclient.setown(createWsDfuClient());
    StringBuffer dfuurl(url.str());
    dfuurl.append("WsDfu");
    dfuclient->addServiceUrl(dfuurl.str());

    const char* username = globals->queryProp("username");
    const char* password = globals->queryProp("password");
    sprayclient->setUsernameToken(username, password, NULL);
    dfuclient->setUsernameToken(username, password, NULL);

}
Beispiel #16
0
int main( int argc, char *argv[]  )
{
#if defined(WIN32) && defined(_DEBUG)
    int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
    tmpFlag |= _CRTDBG_LEAK_CHECK_DF;
    _CrtSetDbgFlag( tmpFlag );
#endif

    InitModuleObjects();

    addAbortHandler(ControlHandler);
    EnableSEHtoExceptionMapping();

    dummyProc();
#ifndef __64BIT__
    // Restrict stack sizes on 32-bit systems
    Thread::setDefaultStackSize(0x10000);   // NB under windows requires linker setting (/stack:)
#endif

#ifdef _WIN32
    Owned<CReleaseMutex> globalNamedMutex;
#endif 

    if (globals)
        globals->Release();

    {
        Owned<IFile> iFile = createIFile("thor.xml");
        globals = iFile->exists() ? createPTree(*iFile, ipt_caseInsensitive) : createPTree("Thor", ipt_caseInsensitive);
    }
    unsigned multiThorMemoryThreshold = 0;

    Owned<IException> unregisterException;
    try
    {
        if (argc==1)
        {
            usage();
            return 1;
        }
        cmdArgs = argv+1;
        mergeCmdParams(globals);
        cmdArgs = argv+1;

        const char *master = globals->queryProp("@MASTER");
        if (!master)
            usage();

        const char *slave = globals->queryProp("@SLAVE");
        if (slave)
        {
            slfEp.set(slave);
            localHostToNIC(slfEp);
        }
        else 
            slfEp.setLocalHost(0);

        mySlaveNum = globals->getPropInt("@SLAVENUM");

        setMachinePortBase(slfEp.port);
        slfEp.port = getMachinePortBase();
        startSlaveLog();

        setSlaveAffinity(globals->getPropInt("@SLAVEPROCESSNUM"));

        startMPServer(getFixedPort(TPORT_mp));
#ifdef USE_MP_LOG
        startLogMsgParentReceiver();
        LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp));
#endif

        SocketEndpoint masterEp(master);
        localHostToNIC(masterEp);
        setMasterPortBase(masterEp.port);
        markNodeCentral(masterEp);
        if (RegisterSelf(masterEp))
        {
            if (globals->getPropBool("Debug/@slaveDaliClient"))
                enableThorSlaveAsDaliClient();

            IDaFileSrvHook *daFileSrvHook = queryDaFileSrvHook();
            if (daFileSrvHook) // probably always installed
                daFileSrvHook->addFilters(globals->queryPropTree("NAS"), &slfEp);

            StringBuffer thorPath;
            globals->getProp("@thorPath", thorPath);
            recursiveCreateDirectory(thorPath.str());
            int err = _chdir(thorPath.str());
            if (err)
            {
                IException *e = makeErrnoExceptionV(-1, "Failed to change dir to '%s'", thorPath.str());
                FLLOG(MCexception(e), thorJob, e);
                throw e;
            }

// Initialization from globals
            setIORetryCount(globals->getPropInt("Debug/@ioRetries")); // default == 0 == off

            StringBuffer str;
            if (globals->getProp("@externalProgDir", str.clear()))
                _mkdir(str.str());
            else
                globals->setProp("@externalProgDir", thorPath);

            const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
            const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
            StringBuffer datadir;
            StringBuffer repdir;
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"data","thor",globals->queryProp("@name"),datadir))
                overrideBaseDirectory = datadir.str();
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir))
                overrideReplicateDirectory = repdir.str();
            if (overrideBaseDirectory&&*overrideBaseDirectory)
                setBaseDirectory(overrideBaseDirectory, false);
            if (overrideReplicateDirectory&&*overrideBaseDirectory)
                setBaseDirectory(overrideReplicateDirectory, true);
            StringBuffer tempDirStr;
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"), tempDirStr))
                globals->setProp("@thorTempDirectory", tempDirStr.str());
            else
                tempDirStr.append(globals->queryProp("@thorTempDirectory"));
            addPathSepChar(tempDirStr).append(getMachinePortBase());

            logDiskSpace(); // Log before temp space is cleared
            SetTempDir(tempDirStr.str(), "thtmp", true);

            useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead"));

            LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR);
            StringBuffer url;
            LOG(MCdebugProgress, thorJob, "Slave %s - temporary dir set to : %s", slfEp.getUrlStr(url).str(), queryTempDir());
#ifdef _WIN32
            ULARGE_INTEGER userfree;
            ULARGE_INTEGER total;
            ULARGE_INTEGER free;
            if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) {
                unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart);
                LOG(MCdebugProgress, thorJob, "Total disk space = %" I64F "d k", total.QuadPart/1000);
                LOG(MCdebugProgress, thorJob, "Free  disk space = %" I64F "d k", free.QuadPart/1000);
                LOG(MCdebugProgress, thorJob, "%d%% disk free\n",pc);
            }
#endif
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"query","thor",globals->queryProp("@name"),str.clear()))
                globals->setProp("@query_so_dir", str.str());
            else
                globals->getProp("@query_so_dir", str.clear());
            if (str.length())
            {
                if (globals->getPropBool("Debug/@dllsToSlaves", true))
                {
                    StringBuffer uniqSoPath;
                    if (PATHSEPCHAR == str.charAt(str.length()-1))
                        uniqSoPath.append(str.length()-1, str.str());
                    else
                        uniqSoPath.append(str);
                    uniqSoPath.append("_").append(getMachinePortBase());
                    str.swapWith(uniqSoPath);
                    globals->setProp("@query_so_dir", str.str());
                }
                PROGLOG("Using querySo directory: %s", str.str());
                recursiveCreateDirectory(str.str());
            }
     
            multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000;
            if (multiThorMemoryThreshold) {
                StringBuffer lgname;
                if (!globals->getProp("@multiThorResourceGroup",lgname))
                    globals->getProp("@nodeGroup",lgname);
                if (lgname.length()) {
                    Owned<ILargeMemLimitNotify> notify = createMultiThorResourceMutex(lgname.str());
                    setMultiThorMemoryNotify(multiThorMemoryThreshold,notify);
                    PROGLOG("Multi-Thor resource limit for %s set to %" I64F "d",lgname.str(),(__int64)multiThorMemoryThreshold);
                }   
                else
                    multiThorMemoryThreshold = 0;
            }
            slaveMain(jobListenerStopped);
        }

        LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK");
    }
    catch (IException *e) 
    {
        if (!jobListenerStopped)
            FLLOG(MCexception(e), thorJob, e,"ThorSlave");
        unregisterException.setown(e);
    }
    ClearTempDirs();

    if (multiThorMemoryThreshold)
        setMultiThorMemoryNotify(0,NULL);
    roxiemem::releaseRoxieHeap();

    if (unregisterException.get())
        UnregisterSelf(unregisterException);

    if (globals->getPropBool("Debug/@slaveDaliClient"))
        disableThorSlaveAsDaliClient();

#ifdef USE_MP_LOG
    stopLogMsgReceivers();
#endif
    stopMPServer();
    ::Release(globals);
    releaseAtoms(); // don't know why we can't use a module_exit to destruct these...

    ExitModuleObjects(); // not necessary, atexit will call, but good for leak checking
    return 0;
}
void GermanStemmer::substitute(StringBuffer& buffer) {
    substCount = 0;

    for ( size_t i = 0; i < buffer.length(); i++ ) {
#ifdef _UCS2
        TCHAR c = buffer.charAt(i);
#else
        unsigned char c = buffer.charAt(i);
#endif
        // Replace the second char of a pair of the equal characters with an asterisk
        if ( i > 0 && c == buffer.charAt ( i - 1 )  ) {
            buffer.setCharAt( i, _T('*') );
        }
        // Substitute Umlauts.
        else if ( c  == 0xe4 ) {
            buffer.setCharAt( i, _T('a') );
        }
        else if ( c == 0xf6 ) {
            buffer.setCharAt( i, _T('o') );
        }
        else if ( c == 0xfc ) {
            buffer.setCharAt( i, _T('u') );
        }
        // Fix bug so that 'ß' at the end of a word is replaced.
        else if ( c == 0xdf ) {
            buffer.setCharAt( i, _T('s') );
            buffer.insert( i + 1, _T('s') );
            substCount++;
        }
        // Take care that at least one character is left left side from the current one
        if ( i < buffer.length() - 1 ) {
            // Masking several common character combinations with an token
            if ( ( i < buffer.length() - 2 ) && c == _T('s') &&
                    buffer.charAt( i + 1 ) == _T('c') && buffer.charAt( i + 2 ) == _T('h') )
            {
                buffer.setCharAt( i, _T('$') );
                buffer.deleteChars( i + 1, i + 3 );
                substCount =+ 2;
            }
            else if ( c == _T('c') && buffer.charAt( i + 1 ) == _T('h') ) {
                buffer.setCharAt( i, 0xa7 ); // section sign in UTF-16
                buffer.deleteCharAt( i + 1 );
                substCount++;
            }
            else if ( c == _T('e') && buffer.charAt( i + 1 ) == _T('i') ) {
                buffer.setCharAt( i, _T('%') );
                buffer.deleteCharAt( i + 1 );
                substCount++;
            }
            else if ( c == _T('i') && buffer.charAt( i + 1 ) == _T('e') ) {
                buffer.setCharAt( i, _T('&') );
                buffer.deleteCharAt( i + 1 );
                substCount++;
            }
            else if ( c == _T('i') && buffer.charAt( i + 1 ) == _T('g') ) {
                buffer.setCharAt( i, _T('#') );
                buffer.deleteCharAt( i + 1 );
                substCount++;
            }
            else if ( c == _T('s') && buffer.charAt( i + 1 ) == _T('t') ) {
                buffer.setCharAt( i, _T('!') );
                buffer.deleteCharAt( i + 1 );
                substCount++;
            }
        }
    }
}
Beispiel #18
0
int HttpClient::sendRequest(int times, HttpStat& stat, StringBuffer& req)
{
    StringBuffer request;
    if(req.length() <= 2)
    {
        throw MakeStringException(-1, "request too short");
    }

    bool endofheaders = false;
    char c0 = req.charAt(0);
    char c1 = req.charAt(1);
    if(c0 == '\n')
        request.append("\r\n");
    else
        request.append(c0);

    if(c1 == '\n')
    {
        if(c0 == '\r')
            request.append(c1);
        else
        {
            request.append("\r\n");
            if(c0 == '\n')
                endofheaders = true;
        }
    }
    else
        request.append(c1);

    unsigned seq = 2;
    while(seq < req.length() && !endofheaders)
    {
        char c = req.charAt(seq);
        if(c == '\n')
        {
            char c1 = req.charAt(seq - 1);
            char c2 = req.charAt(seq - 2);
            if(c1 == '\n' || (c1 == '\r' && c2 == '\n'))
                endofheaders = true;

            if(c1 != '\r')
                request.append("\r\n");
            else
                request.append(c);
        }
        else
            request.append(c);
        seq++;
    }

    if(seq < req.length())
        request.append(req.length() - seq, req.str() + seq);

    if(httptest_tracelevel > 5)
        fprintf(m_ofile, ">>sending out request to %s:%d for %d times\n", m_host.str(), m_port, times);

    unsigned start = msTick();
    int slowest = 0;
    int fastest = 2147483647;
    for(int i = 0; i < times; i++)
    {
        SocketEndpoint ep;
        ep.set(m_host.str(), m_port);
        Owned<ISocket> socket;
        try
        {
            socket.setown(ISocket::connect(ep));
            if(m_use_ssl && m_ssctx.get() != NULL)
            {
                Owned<ISecureSocket> securesocket = m_ssctx->createSecureSocket(socket.getLink());
                int res = securesocket->secure_connect();
                if(res >= 0)
                {
                    socket.set(securesocket.get());
                }
            }
        }
        catch(IException *excpt)
        {
            StringBuffer errMsg;
            DBGLOG("Error connecting to %s:%d - %d:%s", m_host.str(), m_port, excpt->errorCode(), excpt->errorMessage(errMsg).str());
            continue;
        }
        catch(...)
        {
            DBGLOG("can't connect to %s:%d", m_host.str(), m_port);
            continue;
        }

        if(socket.get() == NULL)
        {
            StringBuffer urlstr;
            DBGLOG(">>Can't connect to %s", ep.getUrlStr(urlstr).str());
            continue;
        }

        if(m_delay > 0)
            sleep(m_delay);

        if(httptest_tracelevel > 5)
            fprintf(m_ofile, ">>sending out request:\n");
        if(httptest_tracelevel > 10)
            fprintf(m_ofile, "%s%s%s\n", sepstr, request.str(), sepstr);

        unsigned start1 = msTick();

        socket->write(request.str(), request.length());

        if(httptest_tracelevel > 5)
            fprintf(m_ofile, ">>receiving response:\n");

        StringBuffer buf;
        Owned<IByteOutputStream> ostream = createOutputStream(buf);
        stat.totalresplen += receiveData(socket.get(), ostream.get(), true);
        if(httptest_tracelevel > 10)
            fprintf(m_ofile, "%s%s%s\n", sepstr, buf.str(), sepstr);

        char tmpbuf[256];
        unsigned int sizeread;
        do
        {
            socket->read(tmpbuf, 0, 256, sizeread);
        }
        while(sizeread > 0);

        socket->shutdown();
        socket->close();
        fflush(m_ofile);
        unsigned end1 = msTick();
        int duration = end1 - start1;
        if(duration <= fastest)
            fastest = duration;
        if(duration > slowest)
            slowest = duration;

        if(i % 100 == 0)
            fprintf(stderr, "sent out %d\n", i);
    }

    unsigned end = msTick();
    stat.msecs = end - start;
    stat.numrequests = times;
    stat.totalreqlen = times * request.length();
    stat.slowest = slowest;
    stat.fastest = fastest;

    return 0;
}
Beispiel #19
0
    void GermanStemmer::substitute(StringBuffer& buffer) {
      substCount = 0;

      for ( size_t c = 0; c < buffer.length(); c++ ) {
        // Replace the second char of a pair of the equal characters with an asterisk
        if ( c > 0 && buffer.charAt( c ) == buffer.charAt ( c - 1 )  ) {
          buffer.setCharAt( c, _T('*') );
        }
        // Substitute Umlauts.
        else if ( buffer.charAt( c ) == _T('ä') ) {
          buffer.setCharAt( c, _T('a') );
        }
        else if ( buffer.charAt( c ) == _T('ö') ) {
          buffer.setCharAt( c, _T('o') );
        }
        else if ( buffer.charAt( c ) == _T('ü') ) {
          buffer.setCharAt( c, _T('u') );
        }
        // Fix bug so that 'ß' at the end of a word is replaced.
        else if ( buffer.charAt( c ) == _T('ß') ) {
            buffer.setCharAt( c, _T('s') );
            buffer.insert( c + 1, _T('s') );
            substCount++;
        }
        // Take care that at least one character is left left side from the current one
        if ( c < buffer.length() - 1 ) {
          // Masking several common character combinations with an token
          if ( ( c < buffer.length() - 2 ) && buffer.charAt( c ) == _T('s') &&
            buffer.charAt( c + 1 ) == _T('c') && buffer.charAt( c + 2 ) == _T('h') )
          {
            buffer.setCharAt( c, _T('$') );
            buffer.deleteChars( c + 1, c + 3 );
            substCount =+ 2;
          }
          else if ( buffer.charAt( c ) == _T('c') && buffer.charAt( c + 1 ) == _T('h') ) {
            buffer.setCharAt( c, _T('§') );
            buffer.deleteCharAt( c + 1 );
            substCount++;
          }
          else if ( buffer.charAt( c ) == _T('e') && buffer.charAt( c + 1 ) == _T('i') ) {
            buffer.setCharAt( c, _T('%') );
            buffer.deleteCharAt( c + 1 );
            substCount++;
          }
          else if ( buffer.charAt( c ) == _T('i') && buffer.charAt( c + 1 ) == _T('e') ) {
            buffer.setCharAt( c, _T('&') );
            buffer.deleteCharAt( c + 1 );
            substCount++;
          }
          else if ( buffer.charAt( c ) == _T('i') && buffer.charAt( c + 1 ) == _T('g') ) {
            buffer.setCharAt( c, _T('#') );
            buffer.deleteCharAt( c + 1 );
            substCount++;
          }
          else if ( buffer.charAt( c ) == _T('s') && buffer.charAt( c + 1 ) == _T('t') ) {
            buffer.setCharAt( c, _T('!') );
            buffer.deleteCharAt( c + 1 );
            substCount++;
          }
        }
      }
    }
Beispiel #20
0
int main( int argc, char *argv[]  )
{
#if defined(WIN32) && defined(_DEBUG)
    int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
    tmpFlag |= _CRTDBG_LEAK_CHECK_DF;
    _CrtSetDbgFlag( tmpFlag );
#endif

    InitModuleObjects();

    addAbortHandler(ControlHandler);
    EnableSEHtoExceptionMapping();

    dummyProc();
#ifndef __64BIT__
    Thread::setDefaultStackSize(0x10000);   // NB under windows requires linker setting (/stack:)
#endif

#ifdef _WIN32
    Owned<CReleaseMutex> globalNamedMutex;
#endif 

    if (globals)
        globals->Release();

    {
        Owned<IFile> iFile = createIFile("thor.xml");
        globals = iFile->exists() ? createPTree(*iFile, ipt_caseInsensitive) : createPTree("Thor", ipt_caseInsensitive);
    }
    unsigned multiThorMemoryThreshold = 0;
    try {
        if (argc==1)
        {
            usage();
            return 1;
        }
        cmdArgs = argv+1;
        mergeCmdParams(globals);
        cmdArgs = argv+1;

        const char *master = globals->queryProp("@MASTER");
        if (!master)
            usage();

        const char *slave = globals->queryProp("@SLAVE");
        if (slave)
        {
            slfEp.set(slave);
            localHostToNIC(slfEp);
        }
        else 
            slfEp.setLocalHost(0);

        if (globals->hasProp("@SLAVENUM"))
            mySlaveNum = atoi(globals->queryProp("@SLAVENUM"));
        else
            mySlaveNum = slfEp.port; // shouldn't happen, provided by script

        setMachinePortBase(slfEp.port);
        slfEp.port = getMachinePortBase();
        startSlaveLog();

        startMPServer(getFixedPort(TPORT_mp));
#ifdef USE_MP_LOG
        startLogMsgParentReceiver();
        LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp));
#endif

        SocketEndpoint masterEp(master);
        localHostToNIC(masterEp);
        setMasterPortBase(masterEp.port);
        markNodeCentral(masterEp);
        if (RegisterSelf(masterEp))
        {
#define ISDALICLIENT // JCSMORE plugins *can* access dali - though I think we should probably prohibit somehow.
#ifdef ISDALICLIENT
            const char *daliServers = globals->queryProp("@DALISERVERS");
            if (!daliServers)
            {
                LOG(MCerror, thorJob, "No Dali server list specified\n");
                return 1;
            }
            Owned<IGroup> serverGroup = createIGroup(daliServers, DALI_SERVER_PORT);
            unsigned retry = 0;
            loop {
                try {
                    LOG(MCdebugProgress, thorJob, "calling initClientProcess");
                    initClientProcess(serverGroup,DCR_ThorSlave, getFixedPort(TPORT_mp));
                    break;
                }
                catch (IJSOCK_Exception *e) {
                    if ((e->errorCode()!=JSOCKERR_port_in_use))
                        throw;
                    FLLOG(MCexception(e), thorJob, e,"InitClientProcess");
                    if (retry++>10)
                        throw;
                    e->Release();
                    LOG(MCdebugProgress, thorJob, "Retrying");
                    Sleep(retry*2000);
                }
            }
            setPasswordsFromSDS();
#endif
            IDaFileSrvHook *daFileSrvHook = queryDaFileSrvHook();
            if (daFileSrvHook) // probably always installed
                daFileSrvHook->addSubnetFilters(globals->queryPropTree("NAS"), NULL);

            StringBuffer thorPath;
            globals->getProp("@thorPath", thorPath);
            recursiveCreateDirectory(thorPath.str());
            int err = _chdir(thorPath.str());
            if (err)
            {
                IException *e = MakeErrnoException(-1, "Failed to change dir to '%s'",thorPath.str());
                FLLOG(MCexception(e), thorJob, e);
                throw e;
            }

// Initialization from globals
            setIORetryCount(globals->getPropInt("Debug/@ioRetries")); // default == 0 == off

            StringBuffer str;
            if (globals->getProp("@externalProgDir", str.clear()))
                _mkdir(str.str());
            else
                globals->setProp("@externalProgDir", thorPath);

            const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
            const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
            StringBuffer datadir;
            StringBuffer repdir;
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"data","thor",globals->queryProp("@name"),datadir))
                overrideBaseDirectory = datadir.str();
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir))
                overrideReplicateDirectory = repdir.str();
            if (overrideBaseDirectory&&*overrideBaseDirectory)
                setBaseDirectory(overrideBaseDirectory, false);
            if (overrideReplicateDirectory&&*overrideBaseDirectory)
                setBaseDirectory(overrideReplicateDirectory, true);
            StringBuffer tempdirstr;
            const char *tempdir = globals->queryProp("@thorTempDirectory");
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"),tempdirstr))
                tempdir = tempdirstr.str();
            SetTempDir(tempdir,true);

            useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead"));

            LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR);
            StringBuffer url;
            LOG(MCdebugProgress, thorJob, "Slave %s - temporary dir set to : %s", slfEp.getUrlStr(url).toCharArray(), queryTempDir());
#ifdef _WIN32
            ULARGE_INTEGER userfree;
            ULARGE_INTEGER total;
            ULARGE_INTEGER free;
            if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) {
                unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart);
                LOG(MCdebugProgress, thorJob, "Total disk space = %"I64F"d k", total.QuadPart/1000);
                LOG(MCdebugProgress, thorJob, "Free  disk space = %"I64F"d k", free.QuadPart/1000);
                LOG(MCdebugProgress, thorJob, "%d%% disk free\n",pc);
            }
#endif
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"query","thor",globals->queryProp("@name"),str.clear()))
                globals->setProp("@query_so_dir", str.str());
            else
                globals->getProp("@query_so_dir", str.clear());
            if (str.length())
            {
                if (globals->getPropBool("Debug/@dllsToSlaves", true))
                {
                    StringBuffer uniqSoPath;
                    if (PATHSEPCHAR == str.charAt(str.length()-1))
                        uniqSoPath.append(str.length()-1, str.str());
                    else
                        uniqSoPath.append(str);
                    uniqSoPath.append("_").append(getMachinePortBase());
                    str.swapWith(uniqSoPath);
                    globals->setProp("@query_so_dir", str.str());
                }
                PROGLOG("Using querySo directory: %s", str.str());
                recursiveCreateDirectory(str.str());
            }
     
            multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000;
            if (multiThorMemoryThreshold) {
                StringBuffer lgname;
                if (!globals->getProp("@multiThorResourceGroup",lgname))
                    globals->getProp("@nodeGroup",lgname);
                if (lgname.length()) {
                    Owned<ILargeMemLimitNotify> notify = createMultiThorResourceMutex(lgname.str());
                    setMultiThorMemoryNotify(multiThorMemoryThreshold,notify);
                    PROGLOG("Multi-Thor resource limit for %s set to %"I64F"d",lgname.str(),(__int64)multiThorMemoryThreshold);
                }   
                else
                    multiThorMemoryThreshold = 0;
            }
            slaveMain();
        }

        LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK");
    }
    catch (IException *e) 
    {
        FLLOG(MCexception(e), thorJob, e,"ThorSlave");
        e->Release();
    }
    catch (CATCHALL)
    {
        FLLOG(MCerror, thorJob, "ThorSlave exiting because of uncaught exception");
    }
    ClearTempDirs();

    if (multiThorMemoryThreshold)
        setMultiThorMemoryNotify(0,NULL);
    roxiemem::releaseRoxieHeap();

#ifdef ISDALICLIENT
    closeEnvironment();
    closedownClientProcess();   // dali client closedown
#endif

#ifdef USE_MP_LOG
    stopLogMsgReceivers();
#endif
    stopMPServer();
    ::Release(globals);
    releaseAtoms(); // don't know why we can't use a module_exit to destruct these...

    return 0;
}
Beispiel #21
0
int main(int argc, const char *argv[])
{
    InitModuleObjects();
    EnableSEHtoExceptionMapping();

    NoQuickEditSection xxx;

    Owned<IFile> file = createIFile("dfuserver.xml");
    if (file->exists())
        globals.setown(createPTreeFromXMLFile("dfuserver.xml", ipt_caseInsensitive));
    else
        globals.setown(readOldIni());

    for (unsigned i=1;i<(unsigned)argc;i++) {
        const char *arg = argv[i];
        StringBuffer prop("@");
        StringBuffer val;
        while (*arg && *arg != '=')
            prop.append(*arg++);
        if (*arg) {
            arg++;
            while (isspace(*arg))
                arg++;
            val.append(arg);
            prop.clip();
            val.clip();
            if (prop.length()>1)
                globals->setProp(prop.str(), val.str());
        }
    }
    StringBuffer daliServer;
    StringBuffer queue;
    if (!globals->getProp("@DALISERVERS", daliServer)||!globals->getProp("@QUEUE", queue)) {
        usage();
        globals.clear();
        releaseAtoms();
        return 1;
    }
    Owned<IFile> sentinelFile;
    bool stop = globals->getPropInt("@STOP",0)!=0;
    if (!stop) {
        sentinelFile.setown(createSentinelTarget());
        removeSentinelFile(sentinelFile);

        StringBuffer logname;
        StringBuffer logdir;
        if (!getConfigurationDirectory(globals->queryPropTree("Directories"),"log","dfuserver",globals->queryProp("@name"),logdir))
            globals->getProp("@LOG_DIR", logdir);
        if (logdir.length() && recursiveCreateDirectory(logdir.str()))
            logname.append(logdir);
        else
            appendCurrentDirectory(logname, true);

        if (logname.length() && logname.charAt(logname.length()-1) != PATHSEPCHAR)
            logname.append(PATHSEPCHAR);
        logname.append("dfuserver");
        StringBuffer aliasLogName(logname);
        aliasLogName.append(".log");
        fileMsgHandler = getRollingFileLogMsgHandler(logname.str(), ".log", MSGFIELD_STANDARD, false, true, NULL, aliasLogName.str());
        queryLogMsgManager()->addMonitorOwn(fileMsgHandler, getCategoryLogMsgFilter(MSGAUD_all, MSGCLS_all, 1000));
    }
    StringBuffer ftslogdir;
    if (getConfigurationDirectory(globals->queryPropTree("Directories"),"log","ftslave",globals->queryProp("@name"),ftslogdir)) // NB instance deliberately dfuserver's
        setFtSlaveLogDir(ftslogdir.str());
    setRemoteSpawnSSH(
        globals->queryProp("SSH/@SSHidentityfile"),
        globals->queryProp("SSH/@SSHusername"),
        globals->queryProp("SSH/@SSHpassword"),
        globals->getPropInt("SSH/@SSHtimeout",0),
        globals->getPropInt("SSH/@SSHretries",3),
        "run_");
    bool enableSNMP = globals->getPropInt("@enableSNMP")!=0;
    CSDSServerStatus *serverstatus=NULL;
    Owned<IReplicateServer> replserver;
    try {
        Owned<IGroup> serverGroup = createIGroup(daliServer.str(),DALI_SERVER_PORT);
        initClientProcess(serverGroup, DCR_DfuServer, 0, NULL, NULL, stop?(1000*30):MP_WAIT_FOREVER);
        setPasswordsFromSDS();

        if(!stop)
        {
            if (globals->getPropBool("@enableSysLog",true))
                UseSysLogForOperatorMessages();

            serverstatus = new CSDSServerStatus("DFUserver");
            setDaliServixSocketCaching(true); // speeds up lixux operations

            startLogMsgParentReceiver();    // for auditing
            connectLogMsgManagerToDali();

            engine.setown(createDFUengine());
            addAbortHandler(exitDFUserver);
        }
        const char *q = queue.str();
        loop {
            StringBuffer subq;
            const char *comma = strchr(q,',');
            if (comma)
                subq.append(comma-q,q);
            else
                subq.append(q);
            if (stop) {
                stopDFUserver(subq.str());
            }
            else {
                StringBuffer mask;
                mask.appendf("Queue[@name=\"%s\"][1]",subq.str());
                IPropertyTree *t=serverstatus->queryProperties()->queryPropTree(mask.str());
                if (t)
                    t->setPropInt("@num",t->getPropInt("@num",0)+1);
                else {
                    t = createPTree();
                    t->setProp("@name",subq.str());
                    t->setPropInt("@num",1);
                    serverstatus->queryProperties()->addPropTree("Queue",t);
                }
                serverstatus->commitProperties();
                engine->setDefaultTransferBufferSize((size32_t)globals->getPropInt("@transferBufferSize"));
                engine->startListener(subq.str(),serverstatus);
            }
            if (!comma)
                break;
            q = comma+1;
            if (!*q)
                break;
        }
        q = globals->queryProp("@MONITORQUEUE");
        if (q&&*q) {
            if (stop) {
                stopDFUserver(q);
            }
            else {
                IPropertyTree *t=serverstatus->queryProperties()->addPropTree("MonitorQueue",createPTree());
                t->setProp("@name",q);
                engine->startMonitor(q,serverstatus,globals->getPropInt("@MONITORINTERVAL",60)*1000);
            }
        }
        q = globals->queryProp("@REPLICATEQUEUE");
        if (q&&*q) {
            if (stop) {
                // TBD?
            }
            else {
                replserver.setown(createReplicateServer(q));
                replserver->runServer();
            }
        }
        if (!stop) {
            serverstatus->commitProperties();

            writeSentinelFile(sentinelFile);

            engine->joinListeners();
            if (replserver.get())
                replserver->stopServer();
            LOG(MCprogress, unknownJob, "Exiting");
        }

    }
    catch(IException *e){
        EXCLOG(e, "DFU Server Exception: ");
        e->Release();
    }
    catch (const char *s) {
        WARNLOG("DFU: %s",s);
    }

    delete serverstatus;
    if (stop)
        Sleep(2000);    // give time to stop
    engine.clear();
    globals.clear();
    closeEnvironment();
    closedownClientProcess();
    UseSysLogForOperatorMessages(false);
    setDaliServixSocketCaching(false);
    releaseAtoms();
    return 0;
}
Beispiel #22
0
int main( int argc, char *argv[] )
{
    int res=0;
    if (argc < 3)
    {
        printf
            ("frunssh <nodelistfile> \"command\" [options] \n"
            "    options: -i:<identity-file> \n"
            "             -u:<user> \n"
            "             -n:<number_of_threads>\n"
            "             -t:<connect-timeout-secs>\n"
            "             -a:<connect-attempts>\n"
            "             -d:<working_directory>\n"
            "             -s                -- strict, must match known_hosts\n"
            "             -b                -- background\n"
            "             -pw:<password>    -- INSECURE: requires pssh (NB identity file preferred)\n"
            "             -pe:<password>    -- INSECURE: as -pw except encrypted password\n"
            "             -pl               -- use plink (on windows)\n"
            "             -v                -- verbose, lists commands run\n"
            "             -d                -- dry run (for testing, enables verbose)\n"
            );
        return 255;
    }


    InitModuleObjects();

#ifndef __64BIT__
    // Restrict stack sizes on 32-bit systems
    Thread::setDefaultStackSize(0x10000);   // NB under windows requires linker setting (/stack:)
#endif

    try  {
        StringBuffer logname;
        splitFilename(argv[0], NULL, NULL, &logname, NULL);

        Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator("frunssh");
        lf->setCreateAliasFile(false);
        lf->setMsgFields(MSGFIELD_prefix);
        lf->beginLogging();

        Owned<IFRunSSH> runssh = createFRunSSH();
        runssh->init(argc,argv);
        runssh->exec();
        const StringArray & strArray = runssh->getReplyText();
        const UnsignedArray & unsArray = runssh->getReply();
        for(unsigned i = 0;i < unsArray.ordinality();i++) {
            StringBuffer buf = strArray.item(i);
            // strip newlines off end of string buf
            if (buf.length() && (buf.charAt(buf.length()-1)) == '\n') {
                buf.setLength(buf.length()-1);
                buf.clip();
            }
            if (buf.length())
                PROGLOG("%d: ssh(%d): %s",i+1,unsArray.item(i),buf.str());
        }
    }
    catch(IException *e)
    {
        EXCLOG(e,"frunssh");
        e->Release();
        res=255;
    }
    releaseAtoms();
    return res;
}
bool QueryHelper::doit(FILE * fp)
{
    Owned<IClientWUCreateRequest> creq = wuclient->createWUCreateRequest();
    Owned<IClientWUCreateResponse> cresp = wuclient->WUCreate(creq);
    const IMultiException* excep = &cresp->getExceptions();
    if(excep != NULL && excep->ordinality() > 0)
    {
        StringBuffer msg;
        excep->errorMessage(msg);
        printf("%s\n", msg.str());
        return false;
    }

    IConstECLWorkunit* wu = &cresp->getWorkunit();
    if(!wu)
    {
        printf("can't create workunit\n");
        return false;
    }

    Owned<IClientWUUpdateRequest> ureq = wuclient->createWUUpdateRequest();
    ureq->setWuid(wu->getWuid());

    // Make a workUnit
    StringBuffer jobname;
    if(globals->hasProp("jobname"))
        jobname.append(globals->queryProp("jobname"));

    StringBuffer ecl;
    if (globals->getProp("ecl", ecl))
    {
        if (ecl.length() && ecl.charAt(0)=='@')
        {
            StringBuffer filename(ecl.str()+1);
            ecl.clear().loadFile(filename);
            if (jobname.length() == 0)
                splitFilename(filename, NULL, NULL, &jobname, NULL);
        }
        ureq->setQueryText(ecl.str());
    }
    else if (globals->hasProp("main"))
        ureq->setQueryMainDefinition(globals->queryProp("main"));
    else if (globals->hasProp("attr"))
        ureq->setQueryText(globals->queryProp("attr"));

    if (globals->getPropInt("compileOnly", 0)!=0)
        ureq->setAction(WUActionCompile);
    if (jobname.length())
        ureq->setJobname(jobname);

    IArrayOf<IEspDebugValue> dvals;
    IArrayOf<IEspApplicationValue> avals;
    StringBuffer xmlParams;

    Owned<IPropertyIterator> it = globals->getIterator();
    bool xmlSeen = false;
    ForEach(*it)
    {
        const char * key = it->getPropKey();
        if (key && strlen(key)>1)
        {
            if(key[0] == '-')
            {
                if (key[1] == 'f')
                {
                    Owned<IEspDebugValue> dval = createDebugValue();
                    dval->setName(&key[2]);
                    dval->setValue(globals->queryProp(key));
                    dvals.append(*dval.getLink());
                }
                //All other options are ignored.
            }
            else if(key[0] == '_')
            {
                Owned<IEspApplicationValue> aval = createApplicationValue();
                aval->setApplication("eclplus");
                aval->setName(&key[1]);
                aval->setValue(globals->queryProp(key));
                avals.append(*aval.getLink());
            }
            else if(key[0] == '/')
            {
                if (xmlSeen)
                    throw MakeStringException(0, "query option must not be used with stored or /, and cannot appear more than once");
                // The / form is expected to be used for scalars, so xmlEncode is appropriate.
                // To pass sets or datasets, use the xml= version
                xmlParams.appendf("<%s>", &key[1]);
                encodeXML(globals->queryProp(key), xmlParams);
                xmlParams.appendf("</%s>", &key[1]);
            }
            else if(stricmp(key, "stored")==0)
            {
                if (xmlSeen)
                    throw MakeStringException(0, "query option must not be used with stored or /, and cannot appear more than once");
                const char *xml = globals->queryProp(key);
                try
                {
                    Owned<IPropertyTree> checkValid = createPTreeFromXMLString(xml);
                }
                catch (IException *E)
                {
                    StringBuffer msg;
                    E->errorMessage(msg);
                    E->Release();
                    throw MakeStringException(0, "Invalid xml: %s", msg.str());
                }
                xmlParams.append(xml);
            }
            else if(stricmp(key, "query")==0)
            {
                if (xmlSeen || xmlParams.length())
                    throw MakeStringException(0, "query option must not be used with stored or /, and cannot appear more than once");
                xmlSeen = true;
                StringBuffer xml;
                if (!globals->getProp(key, xml))
                    throw MakeStringException(0, "Invalid value for query= parameter");
                if (xml.length() && xml.charAt(0)=='@')
                {
                    StringBuffer filename(xml.str()+1);
                    xml.clear().loadFile(filename);
                }
                try
                {
                    Owned<IPropertyTree> checkValid = createPTreeFromXMLString(xml);
                }
                catch (IException *E)
                {
                    StringBuffer msg;
                    E->errorMessage(msg);
                    E->Release();
                    throw MakeStringException(0, "Invalid xml: %s", msg.str());
                }
                xmlParams.append(xml);
            }
        }
    }
    if(dvals.length() > 0)
        ureq->setDebugValues(dvals);
    if(avals.length() > 0)
        ureq->setApplicationValues(avals);
    if (xmlParams.length())
    {
        if (!xmlSeen)
        {
            xmlParams.insert(0, "<Query>");
            xmlParams.append("</Query>");
        }
        ureq->setXmlParams(xmlParams);
    }

    Owned<IClientWUUpdateResponse> uresp = wuclient->WUUpdate(ureq);
    const IMultiException* uexcep = &uresp->getExceptions();
    if(uexcep != NULL && uexcep->ordinality() > 0)
    {
        StringBuffer msg;
        uexcep->errorMessage(msg);
        printf("%s\n", msg.str());
        return false;
    }

    // Execute it
    return doSubmitWorkUnit(fp, wu->getWuid(), globals->queryProp("cluster"));
}
Beispiel #24
0
IHqlExpression * HqlCppCaseInfo::buildIndexedMap(BuildCtx & ctx, const CHqlBoundExpr & test)
{
    ITypeInfo * compareType = test.queryType()->queryPromotedType();
    type_t compareTypeCode = compareType->getTypeCode();

    HqlExprArray values;
    IHqlExpression * dft = queryActiveTableSelector();  // value doesn't matter as long as it will not occur
    __int64 lower = getIntValue(lowerTableBound, 0);
    unsigned num = (getIntValue(upperTableBound, 0)-lower)+1;

    CHqlBoundExpr indexExpr;
    switch (compareTypeCode)
    {
        case type_int:
            indexExpr.set(test);
            break;
        case type_string:
            indexExpr.expr.setown(createValue(no_index, makeCharType(), LINK(test.expr), getZero()));
            indexExpr.expr.setown(createValue(no_cast, makeIntType(1, false), LINK(indexExpr.expr)));
            break;
        default:
            throwUnexpectedType(compareType);
    }

    if (useRangeIndex && (num != 1))
        translator.ensureSimpleExpr(ctx, indexExpr);

    OwnedHqlExpr mapped;
    ITypeInfo * retType = resultType;
    //if num == pairs.ordinality() and all results are identical, avoid the table lookup.
    if (allResultsMatch && (num == pairs.ordinality()))
    {
        mapped.set(pairs.item(0).queryChild(1));
    }
    else
    {
        values.ensure(num);
        unsigned idx;
        for (idx = 0; idx < num; idx++)
            values.append(*LINK(dft));

        ForEachItemIn(idx2, pairs)
        {
            IHqlExpression & cur = pairs.item(idx2);
            IValue * value = cur.queryChild(0)->queryValue();
            unsigned replaceIndex;
            switch (compareTypeCode)
            {
            case type_int:
                replaceIndex = (unsigned)(value->getIntValue()-lower);
                break;
            case type_string:
                {
                    StringBuffer temp;
                    value->getStringValue(temp);
                    replaceIndex = (unsigned)((unsigned char)temp.charAt(0)-lower);
                    break;
                }
            default:
                throwUnexpectedType(compareType);
            }

            IHqlExpression * mapTo = cur.queryChild(1);
            if (mapTo->getOperator() != no_constant)
                throwUnexpected();
            if (replaceIndex >= num)
                translator.reportWarning(CategoryIgnored, HQLWRN_CaseCanNeverMatch, "CASE entry %d can never match the test condition", replaceIndex);
            else
                values.replace(*LINK(mapTo),replaceIndex);
        }

        //Now replace the placeholders with the default values.
        for (idx = 0; idx < num; idx++)
        {
            if (&values.item(idx) == dft)
                values.replace(*defaultValue.getLink(),idx);
        }

        // use a var string type to get better C++ generated...
        ITypeInfo * storeType = getArrayElementType(resultType);
        ITypeInfo * listType = makeArrayType(storeType, values.ordinality());
        OwnedHqlExpr lvalues = createValue(no_list, listType, values);

        CHqlBoundExpr boundTable;
        translator.buildExpr(ctx, lvalues, boundTable);

        LinkedHqlExpr tableIndex = indexExpr.expr;
        if (getIntValue(lowerTableBound, 0))
            tableIndex.setown(createValue(no_sub, tableIndex->getType(), LINK(tableIndex), LINK(lowerTableBound)));

        IHqlExpression * ret = createValue(no_index, LINK(retType), LINK(boundTable.expr), LINK(tableIndex));
        mapped.setown(createTranslatedOwned(ret));
    }
    virtual void doWork()
    {
        try
        {
            StringBuffer userId;
            StringBuffer password;
            bool bLinux;

            if (m_sConfigAddress.length() < 1)
            {
                m_pService->getAccountAndPlatformInfo(m_sAddress.str(), userId, password, bLinux);
            }
            else
            {
                m_pService->getAccountAndPlatformInfo(m_sConfigAddress.str(), userId, password, bLinux);
            }

            if (!m_userId.length() || !m_password.length())
            {
                //BUG: 9825 - remote execution on linux needs to use individual accounts
                //use userid/password in ESP context for remote execution...
                if (bLinux)
                {
                    userId.clear();
                    password.clear();
                    m_context.getUserID(userId);
                    m_context.getPassword(password);
                }
            }
            else
            {
                userId.clear().append(m_userId);
                password.clear().append(m_password);
            }

            if (!m_sCommand.length())
                setResultCode(-1);
            else
            {
                IFRunSSH * connection = createFRunSSH();
                connection->init(m_sCommand.str(),NULL,NULL,NULL,5,0);
                connection->exec(m_sAddress.str(),NULL,true);
            }
        }
        // CFRunSSH uses a MakeStringExceptionDirect throw to pass code and result string to caller
        catch(IException* e)
        {
            // errorCode == -1 on successful CFRunSSH execution
            if(e->errorCode() == -1)
                setResultCode(0);
            else
                setResultCode(e->errorCode());
            StringBuffer buf;
            e->errorMessage(buf);
            if (buf.length() && buf.charAt(buf.length() - 1) == '\n') // strip newline
                buf.setLength(buf.length() - 1);
            // set regardless of return
            setResponse(buf.str());
            e->Release();
        }
#ifndef NO_CATCHALL
        catch(...)
        {
            setResponse("An unknown exception occurred!");
            setResultCode(-1);
        }
#endif
    }//doWork()
Beispiel #26
0
int init_main(int argc, char* argv[])
{
    InitModuleObjects();

    Owned<IProperties> inputs = createProperties(true);

    bool interactive = false;

    for (int i = 1; i < argc; i++)
    {
        if (stricmp(argv[i], "-?")==0 || stricmp(argv[i], "-h")==0 || stricmp(argv[i], "-help")==0
             || stricmp(argv[i], "/?")==0 || stricmp(argv[i], "/h")==0)
             usage();
        else if(stricmp(argv[i], "interactive") == 0)
            interactive = true;
        else if (strchr(argv[i],'='))
        {
            inputs->loadProp(argv[i]);
        }
        else
        {
            fprintf(stderr, "Unknown option: %s", argv[i]);
            return 0;
        }
    }

    int result = -1;

#ifdef _WIN32 
    if (!interactive)
        ::SetErrorMode(SEM_NOGPFAULTERRORBOX|SEM_FAILCRITICALERRORS);
#endif

    SET_ESP_SIGNAL_HANDLER(SIGPIPE, brokenpipe_handler);

    bool SEHMappingEnabled = false;

    CEspAbortHandler abortHandler;

    Owned<IFile> sentinelFile = createSentinelTarget();
    removeSentinelFile(sentinelFile);

    Owned<CEspConfig> config;
    Owned<CEspServer> server;
    try
    {
        const char* cfgfile = NULL;
        const char* procname = NULL;
        if(inputs.get())
        {
            if(inputs->hasProp("config"))
                cfgfile = inputs->queryProp("config");
            if(inputs->hasProp("process"))
                procname = inputs->queryProp("process");
        }
        if(!cfgfile || !*cfgfile)
            cfgfile = "esp.xml";

        Owned<IPropertyTree> envpt= createPTreeFromXMLFile(cfgfile, ipt_caseInsensitive);
        Owned<IPropertyTree> procpt = NULL;
        if (envpt)
        {
            envpt->addProp("@config", cfgfile);
            StringBuffer xpath;
            if (procname==NULL || strcmp(procname, ".")==0)
                xpath.appendf("Software/EspProcess[1]");
            else
                xpath.appendf("Software/EspProcess[@name=\"%s\"]", procname);

            DBGLOG("Using ESP configuration section [%s]", xpath.str());
            procpt.set(envpt->queryPropTree(xpath.str()));
            if (!procpt)
                throw MakeStringException(-1, "Config section [%s] not found", xpath.str());
        }
        else
            throw MakeStringException(-1, "Failed to load config file %s", cfgfile);

        StringBuffer logdir;
        if(procpt->hasProp("@name"))
        {
            StringBuffer espNameStr;
            procpt->getProp("@name", espNameStr);
            if (!getConfigurationDirectory(envpt->queryPropTree("Software/Directories"), "log", "esp", espNameStr.str(), logdir))
            {
                logdir.clear();
            }
        }

        const char* build_ver = BUILD_TAG;
        setBuildVersion(build_ver);

        const char* build_level = BUILD_LEVEL;
        setBuildLevel(build_level);

        if(logdir.length() == 0)
        {
            if(procpt->hasProp("@logDir"))
                procpt->getProp("@logDir", logdir);
        }
        if(logdir.length() == 0)
            logdir.append(".");
        if(stricmp(logdir.str(), ".") != 0)
        {
            recursiveCreateDirectory(logdir.str());
        }
        if(logdir.charAt(logdir.length() - 1) != PATHSEPCHAR)
            logdir.append(PATHSEPCHAR);
        
        openEspLogFile(logdir.str(), procpt.get());

        StringBuffer componentfilesDir;
        if(procpt->hasProp("@componentfilesDir"))
            procpt->getProp("@componentfilesDir", componentfilesDir);
        if(componentfilesDir.length() > 0 && strcmp(componentfilesDir.str(), ".") != 0)
        {
            DBGLOG("componentfiles are under %s", componentfilesDir.str());
            setCFD(componentfilesDir.str());
        }

        StringBuffer sehsetting;
        procpt->getProp("@enableSEHMapping", sehsetting);
        if(!interactive && sehsetting.length() > 0 && (stricmp(sehsetting.str(), "true") == 0 || stricmp(sehsetting.str(), "1") == 0))
            SEHMappingEnabled = true;
        if(SEHMappingEnabled)
            EnableSEHtoExceptionMapping();

        CEspConfig* cfg = new CEspConfig(inputs.getLink(), envpt.getLink(), procpt.getLink(), false);
        if(cfg && cfg->isValid())
        {
            config.setown(cfg);
            abortHandler.setConfig(cfg);
        }
    }
    catch(IException* e)
    {
        StringBuffer description;
        ERRLOG("ESP Unhandled IException (%d -- %s)", e->errorCode(), e->errorMessage(description).str());
        e->Release();
        return -1;
    }
    catch (...)
    {
        ERRLOG("ESP Unhandled General Exception.");
        return -1;
    }

    if (config && config->isValid())
    {
        PROGLOG("Configuring Esp Platform...");

        try
        {
            CEspServer *srv = new CEspServer(config);
            if(SEHMappingEnabled)
                srv->setSavedSEHHandler(SEHMappingEnabled);
            server.setown(srv);
            abortHandler.setServer(srv);
            setEspContainer(server.get());

            config->loadAll();
            config->bindServer(*server.get(), *server.get()); 
            
        }
        catch(IException* e)
        {
            StringBuffer description;
            ERRLOG("ESP Unhandled IException (%d -- %s)", e->errorCode(), e->errorMessage(description).str());
            e->Release();
            return -1;
        }
        catch (...)
        {
            ERRLOG("ESP Unhandled General Exception.");
            return -1;
        }

        writeSentinelFile(sentinelFile);
        result = work_main(*config, *server.get());
    }
    else
    {
        ERRLOG("!!! Unable to load ESP configuration.");
    }
    
    return result;
}