/**
* Returns a set of suggestions for the given input touch points. The commitPoint argument indicates
* whether to prematurely commit the suggested words up to the given point for sentence-level
* suggestion.
*
* Note: Currently does not support concurrent calls across threads. Continuous suggestion is
* automatically activated for sequential calls that share the same starting input.
* TODO: Stop detecting continuous suggestion. Start using traverseSession instead.
*/
void Suggest::getSuggestions(ProximityInfo *pInfo, void *traverseSession,
int *inputXs, int *inputYs, int *times, int *pointerIds, int *inputCodePoints,
int inputSize, const float languageWeight,
SuggestionResults *const outSuggestionResults) const {
PROF_OPEN;
PROF_START(0);
const float maxSpatialDistance = TRAVERSAL->getMaxSpatialDistance();
DicTraverseSession *tSession = static_cast<DicTraverseSession *>(traverseSession);
tSession->setupForGetSuggestions(pInfo, inputCodePoints, inputSize, inputXs, inputYs, times,
pointerIds, maxSpatialDistance, TRAVERSAL->getMaxPointerCount());
// TODO: Add the way to evaluate cache
initializeSearch(tSession);
PROF_END(0);
PROF_START(1);
// keep expanding search dicNodes until all have terminated.
while (tSession->getDicTraverseCache()->activeSize() > 0) {
expandCurrentDicNodes(tSession);
tSession->getDicTraverseCache()->advanceActiveDicNodes();
tSession->getDicTraverseCache()->advanceInputIndex(inputSize);
}
PROF_END(1);
PROF_START(2);
SuggestionsOutputUtils::outputSuggestions(
SCORING, tSession, languageWeight, outSuggestionResults);
PROF_END(2);
PROF_CLOSE;
}
void testGlobalNamedCounter(){
PROF_CREATE_COUNTER("GlobalCounter1");
PROF_START("GlobalCounter1");
Sleep(1234);
PROF_STOP("GlobalCounter1");
PROF_START("GlobalCounter1");
Sleep(1234);
PROF_STOP("GlobalCounter1");
PROF_DESTROY_COUNTER("GlobalCounter1");
}
void testGlobalNamedCounterFlush(){
PROF_CREATE_COUNTER("GlobalCounter2");
PROF_START("GlobalCounter2");
Sleep(1234);
PROF_STOP("GlobalCounter2");
PROF_FLUSH_COUNTER("GlobalCounter2","Step1");
PROF_START("GlobalCounter2");
Sleep(1234);
PROF_STOP("GlobalCounter2");
PROF_DESTROY_COUNTER("GlobalCounter2");
}
void CSyncSource::syncServerChanges()
{
LOG(INFO) + "Sync server changes source ID :" + getID();
while( getSync().isContinueSync() )
{
setCurPageCount(0);
String strUrl = getProtocol().getServerQueryUrl("");
String strQuery = getProtocol().getServerQueryBody(getName(), getSync().getClientID(), getSync().getSyncPageSize());
if ( !m_bTokenFromDB && getToken() > 1 )
strQuery += "&token=" + convertToStringA(getToken());
LOG(INFO) + "Pull changes from server. Url: " + (strUrl+strQuery);
PROF_START("Net");
NetResponse(resp,getNet().pullData(strUrl+strQuery, &getSync()));
PROF_STOP("Net");
if ( !resp.isOK() )
{
getSync().stopSync();
m_nErrCode = RhoAppAdapter.getErrorFromResponse(resp);
m_strError = resp.getCharData();
continue;
}
const char* szData = resp.getCharData();
//const char* szData = "[{\"version\":3},{\"token\":\"\"},{\"count\":0},{\"progress_count\":28},{\"total_count\":28},{\"source-error\":{\"login-error\":{\"message\":\"s currently connected from another machine\"}}}]";
//const char* szData = "[{\"version\":3},{\"token\":\"\"},{\"count\":0},{\"progress_count\":0},{\"total_count\":0},{\"create-error\":{\"0_broken_object_id\":{\"name\":\"wrongname\",\"an_attribute\":\"error create\"},\"0_broken_object_id-error\":{\"message\":\"error create\"}}}]";
//const char* szData = "[{\"version\":3},{\"token\":\"35639160294387\"},{\"count\":3},{\"progress_count\":0},{\"total_count\":3},{\"metadata\":\"{\\\"foo\\\":\\\"bar\\\"}\",\"insert\":{\"1\":{\"price\":\"199.99\",\"brand\":\"Apple\",\"name\":\"iPhone\"}}}]";
//LOG(INFO) + szData;
PROF_START("Parse");
CJSONArrayIterator oJsonArr(szData);
PROF_STOP("Parse");
processServerResponse_ver3(oJsonArr);
if (getSync().getSourceOptions().getBoolProperty(getID(), "pass_through"))
processToken(0);
if ( getToken() == 0 )
break;
}
if ( getSync().isSchemaChanged() )
getSync().stopSync();
}
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
PROF_OPEN;
PROF_START(66);
const jsize sourceDirUtf8Length = env->GetStringUTFLength(sourceDir);
if (sourceDirUtf8Length <= 0) {
AKLOGE("DICT: Can't get sourceDir string");
return 0;
}
char sourceDirChars[sourceDirUtf8Length + 1];
env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
sourceDirChars[sourceDirUtf8Length] = '\0';
DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy =
DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(
sourceDirChars, static_cast<int>(dictOffset), static_cast<int>(dictSize),
isUpdatable == JNI_TRUE);
if (!dictionaryStructureWithBufferPolicy) {
return 0;
}
Dictionary *const dictionary = new Dictionary(env, dictionaryStructureWithBufferPolicy);
PROF_END(66);
PROF_CLOSE;
return reinterpret_cast<jlong>(dictionary);
}
int timer_pdomain(pdomain_t* _d)
{
struct presentity* presentity, *t;
time_t start, stop;
PROF_START(pa_timer_pdomain)
lock_pdomain(_d);
start = time(NULL);
presentity = _d->first;
while(presentity) {
if (timer_presentity(presentity) < 0) {
LOG(L_ERR, "timer_pdomain(): Error in timer_pdomain\n");
unlock_pdomain(_d);
PROF_STOP(pa_timer_pdomain)
return -1;
}
/* Remove the entire record
* if it is empty
*/
if ( (!presentity->first_watcher) &&
(!presentity->first_winfo_watcher) &&
(!presentity->data.first_tuple) &&
(!presentity->data.first_note) &&
(!presentity->data.first_unknown_element) &&
(!presentity->first_qsa_subscription) &&
(presentity->ref_cnt == 0)) {
LOG(L_DBG, "timer_pdomain(): removing empty presentity\n");
t = presentity;
presentity = presentity->next;
release_presentity(t);
} else {
presentity = presentity->next;
}
}
bool CHttpServer::send_file(String const &path, HeaderList const &hdrs)
{
String fullPath = CFilePath::normalizePath(path);
if (String_startsWith(fullPath,"/app/db/db-files") )
fullPath = CFilePath::join( rho_native_rhodbpath(), path.substr(4) );
else if (fullPath.find(m_root) != 0 && fullPath.find(m_strRhoRoot) != 0 && fullPath.find(m_strRuntimeRoot) != 0 && fullPath.find(m_userroot) != 0 && fullPath.find(m_strRhoUserRoot) != 0)
fullPath = CFilePath::join( m_root, path );
struct stat st;
bool bCheckExist = true;
#ifdef RHODES_EMULATOR
String strPlatform = RHOSIMCONF().getString("platform");
if ( strPlatform.length() > 0 )
{
String fullPath1 = fullPath;
int nDot = fullPath1.rfind('.');
if ( nDot >= 0 )
fullPath1.insert(nDot, String(".") + strPlatform);
else
fullPath1 += String(".") + strPlatform;
if (stat(fullPath1.c_str(), &st) == 0 && S_ISREG(st.st_mode))
{
fullPath = fullPath1;
bCheckExist = false;
}
}
#endif
bool doesNotExists = bCheckExist && (stat(fullPath.c_str(), &st) != 0 || !S_ISREG(st.st_mode));
if ( doesNotExists ) {
// looking for files at 'rho/apps' at runtime folder
fullPath = CFilePath::join( m_strRuntimeRoot, path );
}
if (verbose) RAWTRACE1("Sending file %s...", fullPath.c_str());
if ( doesNotExists ) {
if ( stat(fullPath.c_str(), &st) != 0 || !S_ISREG(st.st_mode) ) {
doesNotExists = true;
}else
doesNotExists = false;
}
#ifdef RHODES_EMULATOR
if ( doesNotExists )
{
CTokenizer oTokenizer( RHOSIMCONF().getString("ext_path"), ";" );
while (oTokenizer.hasMoreTokens())
{
String tok = oTokenizer.nextToken();
tok = String_trim(tok);
String fullPath1 = CFilePath::join( tok, path );
if (stat(fullPath1.c_str(), &st) == 0 && S_ISREG(st.st_mode))
{
fullPath = fullPath1;
doesNotExists = false;
break;
}
}
}
#endif
if ( doesNotExists ) {
if (verbose) RAWLOG_ERROR1("The file %s was not found", path.c_str());
String error = "<!DOCTYPE html><html><font size=\"+4\"><h2>404 Not Found.</h2> The file " + path + " was not found.</font></html>";
send_response(create_response("404 Not Found",error));
return false;
}
PROF_START("LOW_FILE");
FILE *fp = fopen(fullPath.c_str(), "rb");
PROF_STOP("LOW_FILE");
if (!fp) {
if (verbose) RAWLOG_ERROR1("The file %s could not be opened", path.c_str());
String error = "<!DOCTYPE html><html><font size=\"+4\"><h2>404 Not Found.</h2> The file " + path + " could not be opened.</font></html";
send_response(create_response("404 Not Found",error));
return false;
}
HeaderList headers;
// Detect mime type
headers.push_back(Header("Content-Type", get_mime_type(path)));
if ( String_startsWith(path, "/public") )
{
headers.push_back(Header("Expires", "Thu, 15 Apr 2020 20:00:00 GMT") );
headers.push_back(Header("Cache-Control", "max-age=2592000") );
}
// Content length
char* buf = new char[FILE_BUF_SIZE];
String start_line;
size_t file_size = st.st_size;
size_t range_begin = 0, range_end = file_size - 1;
size_t content_size = file_size;
if (parse_range(hdrs, &range_begin, &range_end))
{
if (range_end >= file_size)
range_end = file_size - 1;
if (range_begin >= range_end)
range_begin = range_end - 1;
content_size = range_end - range_begin + 1;
if (fseek(fp, range_begin, SEEK_SET) == -1) {
RAWLOG_ERROR1("Can not seek to specified range start: %lu", (unsigned long)range_begin);
snprintf(buf, FILE_BUF_SIZE, "bytes */%lu", (unsigned long)file_size);
headers.push_back(Header("Content-Range", buf));
send_response(create_response("416 Request Range Not Satisfiable",headers));
fclose(fp);
delete[] buf;
return false;
}
snprintf(buf, FILE_BUF_SIZE, "bytes %lu-%lu/%lu", (unsigned long)range_begin,
(unsigned long)range_end, (unsigned long)file_size);
headers.push_back(Header("Content-Range", buf));
start_line = "206 Partial Content";
}
else {
start_line = "200 OK";
}
snprintf(buf, FILE_BUF_SIZE, "%lu", (unsigned long)content_size);
headers.push_back(Header("Content-Length", buf));
// Send headers
if (!send_response(create_response(start_line, headers))) {
if (verbose) RAWLOG_ERROR1("Can not send headers while sending file %s", path.c_str());
fclose(fp);
delete[] buf;
return false;
}
// Send body
for (size_t start = range_begin; start < range_end + 1;) {
size_t need_to_read = range_end - start + 1;
if (need_to_read == 0)
break;
if (need_to_read > FILE_BUF_SIZE)
need_to_read = FILE_BUF_SIZE;
PROF_START("LOW_FILE");
size_t n = fread(buf, 1, need_to_read, fp);//fread(buf, 1, need_to_read, fp);
PROF_STOP("LOW_FILE");
if (n < need_to_read) {
if (ferror(fp) ) {
if (verbose) RAWLOG_ERROR2("Can not read part of file (at position %lu): %s", (unsigned long)start, strerror(errno));
} else if ( feof(fp) ) {
if (verbose) RAWLOG_ERROR1("End of file reached, but we expect data (%lu bytes)", (unsigned long)need_to_read);
}
fclose(fp);
delete[] buf;
return false;
}
start += n;
if (!send_response_body(String(buf, n))) {
if (verbose) RAWLOG_ERROR1("Can not send part of data while sending file %s", path.c_str());
fclose(fp);
delete[] buf;
return false;
}
}
PROF_START("LOW_FILE");
fclose(fp);
PROF_STOP("LOW_FILE");
delete[] buf;
if (verbose) RAWTRACE1("File %s was sent successfully", path.c_str());
return false;
}
static jint latinime_BinaryDictionary_open(JNIEnv *env, jobject object,
jstring sourceDir, jlong dictOffset, jlong dictSize,
jint typedLetterMultiplier, jint fullWordMultiplier, jint maxWordLength, jint maxWords,
jint maxAlternatives) {
PROF_OPEN;
PROF_START(66);
const char *sourceDirChars = env->GetStringUTFChars(sourceDir, NULL);
if (sourceDirChars == NULL) {
LOGE("DICT: Can't get sourceDir string");
return 0;
}
int fd = 0;
void *dictBuf = NULL;
int adjust = 0;
#ifdef USE_MMAP_FOR_DICTIONARY
/* mmap version */
fd = open(sourceDirChars, O_RDONLY);
if (fd < 0) {
LOGE("DICT: Can't open sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno);
return 0;
}
int pagesize = getpagesize();
adjust = dictOffset % pagesize;
int adjDictOffset = dictOffset - adjust;
int adjDictSize = dictSize + adjust;
dictBuf = mmap(NULL, sizeof(char) * adjDictSize, PROT_READ, MAP_PRIVATE, fd, adjDictOffset);
if (dictBuf == MAP_FAILED) {
LOGE("DICT: Can't mmap dictionary. errno=%d", errno);
return 0;
}
dictBuf = (void *)((char *)dictBuf + adjust);
#else // USE_MMAP_FOR_DICTIONARY
/* malloc version */
FILE *file = NULL;
file = fopen(sourceDirChars, "rb");
if (file == NULL) {
LOGE("DICT: Can't fopen sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno);
return 0;
}
dictBuf = malloc(sizeof(char) * dictSize);
if (!dictBuf) {
LOGE("DICT: Can't allocate memory region for dictionary. errno=%d", errno);
return 0;
}
int ret = fseek(file, (long)dictOffset, SEEK_SET);
if (ret != 0) {
LOGE("DICT: Failure in fseek. ret=%d errno=%d", ret, errno);
return 0;
}
ret = fread(dictBuf, sizeof(char) * dictSize, 1, file);
if (ret != 1) {
LOGE("DICT: Failure in fread. ret=%d errno=%d", ret, errno);
return 0;
}
ret = fclose(file);
if (ret != 0) {
LOGE("DICT: Failure in fclose. ret=%d errno=%d", ret, errno);
return 0;
}
#endif // USE_MMAP_FOR_DICTIONARY
env->ReleaseStringUTFChars(sourceDir, sourceDirChars);
if (!dictBuf) {
LOGE("DICT: dictBuf is null");
return 0;
}
Dictionary *dictionary = NULL;
if (BinaryFormat::UNKNOWN_FORMAT == BinaryFormat::detectFormat((uint8_t*)dictBuf)) {
LOGE("DICT: dictionary format is unknown, bad magic number");
#ifdef USE_MMAP_FOR_DICTIONARY
releaseDictBuf(((char*)dictBuf) - adjust, adjDictSize, fd);
#else // USE_MMAP_FOR_DICTIONARY
releaseDictBuf(dictBuf, 0, 0);
#endif // USE_MMAP_FOR_DICTIONARY
} else {
dictionary = new Dictionary(dictBuf, dictSize, fd, adjust, typedLetterMultiplier,
fullWordMultiplier, maxWordLength, maxWords, maxAlternatives);
}
PROF_END(66);
PROF_CLOSE;
return (jint)dictionary;
}
void CSyncSource::processServerData(const char* szData)
{
PROF_START("Parse");
CJSONArrayIterator oJsonArr(szData);
PROF_STOP("Parse");
PROF_START("Data1");
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("error") )
{
m_strError = oJsonArr.getCurItem().getString("error");
m_nErrCode = RhoRuby.ERR_CUSTOMSYNCSERVER;
getSync().stopSync();
return;
}
if ( !oJsonArr.isEnd() )
{
setCurPageCount(oJsonArr.getCurItem().getInt("count"));
oJsonArr.next();
}
int nVersion = 0;
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("version") )
{
nVersion = oJsonArr.getCurItem().getInt("version");
oJsonArr.next();
}
if ( nVersion != getSync().SYNC_VERSION() )
{
LOG(ERROR) + "Sync server send data with incompatible version. Client version: " + convertToStringA(getSync().SYNC_VERSION()) +
"; Server response version: " + convertToStringA(nVersion) + ". Source name: " + getName();
getSync().stopSync();
m_nErrCode = RhoRuby.ERR_SYNCVERSION;
return;
}
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("rt") )
{
setRefreshTime(oJsonArr.getCurItem().getInt("rt"));
oJsonArr.next();
}
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("total_count") )
{
setTotalCount(oJsonArr.getCurItem().getInt("total_count"));
oJsonArr.next();
}
//if ( getServerObjectsCount() == 0 )
// getNotify().fireSyncNotification(this, false, RhoRuby.ERR_NONE, "");
if ( !oJsonArr.isEnd() )
{
processToken(oJsonArr.getCurItem().getUInt64("token"));
oJsonArr.next();
}else if ( getCurPageCount() == 0 )
{
//oo conflicts
getDB().executeSQL("DELETE FROM changed_values where source_id=? and sent>=3", getID() );
//
processToken(0);
}
LOG(INFO) + "Got " + getCurPageCount() + "(Processed: " + getServerObjectsCount() + ") records of " + getTotalCount() + " from server. Source: " + getName()
+ ". Version: " + nVersion;
PROF_STOP("Data1");
if ( !oJsonArr.isEnd() && getSync().isContinueSync() )
{
PROF_START("Data");
//TODO: support DBExceptions
getDB().startTransaction();
int nSavedPos = oJsonArr.getCurPos();
setSyncServerDataPass(edpNone);
processServerData_Ver1(oJsonArr);
setSyncServerDataPass(edpDeleteObjects);
oJsonArr.reset(nSavedPos);
processServerData_Ver1(oJsonArr);
PROF_STOP("Data");
PROF_START("DB");
getDB().endTransaction();
PROF_STOP("DB");
getNotify().fireObjectsNotification();
}
PROF_START("Data1");
if ( getCurPageCount() > 0 )
getNotify().fireSyncNotification(this, false, RhoRuby.ERR_NONE, "");
PROF_STOP("Data1");
}
void CSyncSource::syncServerChanges()
{
LOG(INFO) + "Sync server changes source ID :" + getID();
while( getSync().isContinueSync() )
{
setCurPageCount(0);
String strUrl = getUrl();
if ( m_strAction.length() > 0 )
strUrl = CFilePath::join( strUrl, m_strAction);
String strQuery = getSync().SYNC_SOURCE_FORMAT() + "&client_id=" + getSync().getClientID() +
"&p_size=" + getSync().SYNC_PAGE_SIZE() + "&version=" + convertToStringA(getSync().SYNC_VERSION());
if ( m_strParams.length() > 0 )
strQuery += m_strParams;
if( m_strUrlParams.length() > 0 )
strQuery += "&" + m_strUrlParams;
if ( getAskParams().length() > 0 )
{
strUrl += getSync().SYNC_ASK_ACTION();
strQuery += "&question=" + getAskParams();
}
if ( !m_bTokenFromDB && getToken() > 1 )
strQuery += "&ack_token=" + convertToStringA(getToken());
LOG(INFO) + "Pull changes from server. Url: " + (strUrl+strQuery);
PROF_START("Net");
NetResponse(resp,getNet().pullData(strUrl+strQuery, &getSync()));
PROF_STOP("Net");
if ( !resp.isOK() )
{
getSync().stopSync();
if (resp.isResponseRecieved())
m_nErrCode = RhoRuby.ERR_REMOTESERVER;
else
m_nErrCode = RhoRuby.ERR_NETWORK;
continue;
}
processServerData(resp.getCharData());
//String strData =
//"[{count:10},{version:1},{total_count: 5425},{token: 123},{s:\"RhoDeleteSource\",ol:[{o:\"rho_del_obj\",av:[{i:55550425},{i:75665819},{i:338165272},{i:402396629},{i:521753981},{i:664143530},{i:678116186},{i:831092394},{i:956041217},{i:970452458}]}]}]";
/*"[{count: 124},{version: 1},{total_count: 5425},{token: 123},"
"{s:\"Product\",ol:["
"{oo:\"123\",o:\"2ed2e0c7-8c4c-99c6-1b37-498d250bb8e7\",av:["
"{a:\"first_name\",i:47354289,v:\"Lars. \n\n Burgess\", t:\"blob\"},"
"{a:\"second_name\",i:55555,v:\"Burgess\"}]},"
"{oo:\"456\", e:\"Something went wrong creating this record on the backend: code 7\"}"
"]}]"; */
/*"[{count: 1},{version: 1},{total_count: 1},{token: 123},"
"{s:\"Product\",ol:["
"{oo:\"94\", e:\"Something went wrong creating this record on the backend: code 7\"}"
"]}]";*/
/*"[{count: 1},{version: 1},{total_count: 1},{token: 123},"
"{s:\"Product\",ol:["
"{o:\"94\", av:["
"{a:\"TEST\",i:55555,v:\"Geny\"}]},"
"]}]";
//u:\"query\",
processServerData(strData.c_str()); */
if ( getAskParams().length() > 0 || getCurPageCount() == 0 )
break;
}
}
RHO_GLOBAL void JNICALL Java_com_rhomobile_rhodes_Logger_profStart
(JNIEnv *env, jclass, jstring tag)
{
PROF_START(rho_cast<std::string>(env, tag).c_str());
}
void testLocalNamedCounter(){
PROF_START("LocalCounter1");
Sleep(2126);
PROF_STOP("LocalCounter1");
}
void CSyncSource::processServerResponse_ver3(CJSONArrayIterator& oJsonArr)
{
PROF_START("Data1");
int nVersion = 0;
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("version") )
{
nVersion = oJsonArr.getCurItem().getInt("version");
oJsonArr.next();
}
if ( nVersion != getProtocol().getVersion() )
{
LOG(ERROR) + "Sync server send data with incompatible version. Client version: " + convertToStringA(getProtocol().getVersion()) +
"; Server response version: " + convertToStringA(nVersion) + ". Source name: " + getName();
getSync().stopSync();
m_nErrCode = RhoAppAdapter.ERR_SYNCVERSION;
return;
}
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("token"))
{
processToken(oJsonArr.getCurItem().getUInt64("token"));
oJsonArr.next();
}
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("source") )
{
//skip it. it uses in search only
oJsonArr.next();
}
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("count") )
{
setCurPageCount(oJsonArr.getCurItem().getInt("count"));
oJsonArr.next();
}
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("refresh_time") )
{
setRefreshTime(oJsonArr.getCurItem().getInt("refresh_time"));
oJsonArr.next();
}
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("progress_count") )
{
//TODO: progress_count
//setTotalCount(oJsonArr.getCurItem().getInt("progress_count"));
oJsonArr.next();
}
if ( !oJsonArr.isEnd() && oJsonArr.getCurItem().hasName("total_count") )
{
setTotalCount(oJsonArr.getCurItem().getInt("total_count"));
oJsonArr.next();
}
//if ( getServerObjectsCount() == 0 )
// getNotify().fireSyncNotification(this, false, RhoAppAdapter.ERR_NONE, "");
if ( getToken() == 0 )
{
//oo conflicts
getDB().executeSQL("DELETE FROM changed_values where source_id=? and sent>=3", getID() );
//
}
LOG(INFO) + "Got " + getCurPageCount() + "(Processed: " + getServerObjectsCount() + ") records of " + getTotalCount() + " from server. Source: " + getName()
+ ". Version: " + nVersion;
PROF_STOP("Data1");
if ( !oJsonArr.isEnd() && getSync().isContinueSync() )
{
CJSONEntry oCmds = oJsonArr.getCurItem();
PROF_START("Data");
//TODO: use isUIWaitDB inside processSyncCommand
// if ( getDB().isUIWaitDB() )
// {
// LOG(INFO) + "Commit transaction because of UI request.";
// getDB().endTransaction();
// getDB().startTransaction();
// }
if ( oCmds.hasName("schema-changed") )
{
getSync().setSchemaChanged(true);
}else if ( oCmds.hasName("source-error") )
{
CJSONEntry errSrc = oCmds.getEntry("source-error");
CJSONStructIterator errIter(errSrc);
for( ; !errIter.isEnd(); errIter.next() )
{
m_nErrCode = RhoAppAdapter.ERR_CUSTOMSYNCSERVER;
m_strError = errIter.getCurValue().getString("message");
m_strErrorType = errIter.getCurKey();
}
}else if ( oCmds.hasName("search-error") )
{
CJSONEntry errSrc = oCmds.getEntry("search-error");
CJSONStructIterator errIter(errSrc);
for( ; !errIter.isEnd(); errIter.next() )
{
m_nErrCode = RhoAppAdapter.ERR_CUSTOMSYNCSERVER;
m_strError = errIter.getCurValue().getString("message");
m_strErrorType = errIter.getCurKey();
}
}else if ( oCmds.hasName("create-error") )
{
m_nErrCode = RhoAppAdapter.ERR_CUSTOMSYNCSERVER;
m_strErrorType = "create-error";
}else if ( oCmds.hasName("update-error") )
{
m_nErrCode = RhoAppAdapter.ERR_CUSTOMSYNCSERVER;
m_strErrorType = "update-error";
}else if ( oCmds.hasName("delete-error") )
{
m_nErrCode = RhoAppAdapter.ERR_CUSTOMSYNCSERVER;
m_strErrorType = "delete-error";
}else
{
getDB().startTransaction();
if (getSync().getSourceOptions().getBoolProperty(getID(), "pass_through"))
{
if ( m_bSchemaSource )
getDB().executeSQL( (String("DELETE FROM ") + getName()).c_str() );
else
getDB().executeSQL("DELETE FROM object_values WHERE source_id=?", getID() );
}
if ( oCmds.hasName("metadata") && getSync().isContinueSync() )
{
String strMetadata = oCmds.getString("metadata");
getDB().executeSQL("UPDATE sources SET metadata=? WHERE source_id=?", strMetadata, getID() );
}
if ( oCmds.hasName("links") && getSync().isContinueSync() )
processSyncCommand("links", oCmds.getEntry("links") );
if ( oCmds.hasName("delete") && getSync().isContinueSync() )
processSyncCommand("delete", oCmds.getEntry("delete") );
if ( oCmds.hasName("insert") && getSync().isContinueSync() )
processSyncCommand("insert", oCmds.getEntry("insert") );
PROF_STOP("Data");
PROF_START("DB");
getDB().endTransaction();
PROF_STOP("DB");
getNotify().fireObjectsNotification();
}
}
PROF_START("Data1");
if ( getCurPageCount() > 0 )
getNotify().fireSyncNotification(this, false, RhoAppAdapter.ERR_NONE, "");
PROF_STOP("Data1");
}
void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
const int *xcoordinates, const int *ycoordinates, const int *codes,
const int inputLength, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
const bool useFullEditDistance, Correction *correction, WordsPriorityQueuePool *queuePool) {
PROF_OPEN;
PROF_START(0);
PROF_END(0);
PROF_START(1);
getOneWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, bigramMap, bigramFilter,
useFullEditDistance, inputLength, correction, queuePool);
PROF_END(1);
PROF_START(2);
// Note: This line is intentionally left blank
PROF_END(2);
PROF_START(3);
// Note: This line is intentionally left blank
PROF_END(3);
PROF_START(4);
bool hasAutoCorrectionCandidate = false;
WordsPriorityQueue* masterQueue = queuePool->getMasterQueue();
if (masterQueue->size() > 0) {
float nsForMaster = masterQueue->getHighestNormalizedScore(
proximityInfo->getPrimaryInputWord(), inputLength, 0, 0, 0);
hasAutoCorrectionCandidate = (nsForMaster > START_TWO_WORDS_CORRECTION_THRESHOLD);
}
PROF_END(4);
PROF_START(5);
// Multiple word suggestions
if (SUGGEST_MULTIPLE_WORDS
&& inputLength >= MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION) {
getSplitMultipleWordsSuggestions(proximityInfo, xcoordinates, ycoordinates, codes,
useFullEditDistance, inputLength, correction, queuePool,
hasAutoCorrectionCandidate);
}
PROF_END(5);
PROF_START(6);
// Note: This line is intentionally left blank
PROF_END(6);
if (DEBUG_DICT) {
queuePool->dumpSubQueue1TopSuggestions();
for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) {
WordsPriorityQueue* queue = queuePool->getSubQueue(FIRST_WORD_INDEX, i);
if (queue->size() > 0) {
WordsPriorityQueue::SuggestedWord* sw = queue->top();
const int score = sw->mScore;
const unsigned short* word = sw->mWord;
const int wordLength = sw->mWordLength;
float ns = Correction::RankingAlgorithm::calcNormalizedScore(
proximityInfo->getPrimaryInputWord(), i, word, wordLength, score);
ns += 0;
AKLOGI("--- TOP SUB WORDS for %d --- %d %f [%d]", i, score, ns,
(ns > TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD));
DUMP_WORD(proximityInfo->getPrimaryInputWord(), i);
DUMP_WORD(word, wordLength);
}
}
}
}
// bigramMap contains the association <bigram address> -> <bigram frequency>
// bigramFilter is a bloom filter for fast rejection: see functions setInFilter and isInFilter
// in bigram_dictionary.cpp
int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo,
WordsPriorityQueuePool *queuePool, Correction *correction, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int codesSize,
const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
const bool useFullEditDistance, unsigned short *outWords, int *frequencies) {
queuePool->clearAll();
Correction* masterCorrection = correction;
correction->resetCorrection();
if (BinaryFormat::REQUIRES_GERMAN_UMLAUT_PROCESSING & FLAGS)
{ // Incrementally tune the word and try all possibilities
int codesBuffer[getCodesBufferSize(codes, codesSize)];
int xCoordinatesBuffer[codesSize];
int yCoordinatesBuffer[codesSize];
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter,
useFullEditDistance, codes, codesSize, 0, codesBuffer, masterCorrection,
queuePool, GERMAN_UMLAUT_DIGRAPHS,
sizeof(GERMAN_UMLAUT_DIGRAPHS) / sizeof(GERMAN_UMLAUT_DIGRAPHS[0]));
} else if (BinaryFormat::REQUIRES_FRENCH_LIGATURES_PROCESSING & FLAGS) {
int codesBuffer[getCodesBufferSize(codes, codesSize)];
int xCoordinatesBuffer[codesSize];
int yCoordinatesBuffer[codesSize];
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter,
useFullEditDistance, codes, codesSize, 0, codesBuffer, masterCorrection,
queuePool, FRENCH_LIGATURES_DIGRAPHS,
sizeof(FRENCH_LIGATURES_DIGRAPHS) / sizeof(FRENCH_LIGATURES_DIGRAPHS[0]));
} else { // Normal processing
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize,
bigramMap, bigramFilter, useFullEditDistance, masterCorrection, queuePool);
}
PROF_START(20);
if (DEBUG_DICT) {
float ns = queuePool->getMasterQueue()->getHighestNormalizedScore(
proximityInfo->getPrimaryInputWord(), codesSize, 0, 0, 0);
ns += 0;
AKLOGI("Max normalized score = %f", ns);
}
const int suggestedWordsCount =
queuePool->getMasterQueue()->outputSuggestions(
proximityInfo->getPrimaryInputWord(), codesSize, frequencies, outWords);
if (DEBUG_DICT) {
float ns = queuePool->getMasterQueue()->getHighestNormalizedScore(
proximityInfo->getPrimaryInputWord(), codesSize, 0, 0, 0);
ns += 0;
AKLOGI("Returning %d words", suggestedWordsCount);
/// Print the returned words
for (int j = 0; j < suggestedWordsCount; ++j) {
short unsigned int* w = outWords + j * MAX_WORD_LENGTH;
char s[MAX_WORD_LENGTH];
for (int i = 0; i <= MAX_WORD_LENGTH; i++) s[i] = w[i];
(void)s;
AKLOGI("%s %i", s, frequencies[j]);
}
}
PROF_END(20);
PROF_CLOSE;
return suggestedWordsCount;
}
bool CHttpServer::decide(String const &method, String const &arg_uri, String const &query,
HeaderList const &headers, String const &body/*, IResponseSender& respSender*/ )
{
if (verbose) RAWTRACE1("Decide what to do with uri %s", arg_uri.c_str());
callback_t callback = registered(arg_uri);
if (callback) {
if (verbose) RAWTRACE1("Uri %s is registered callback, so handle it appropriately", arg_uri.c_str());
if ( callback == rho_http_ruby_proc_callback )
call_ruby_proc( query, body );
else
callback(this, query.length() ? query : body);
return false;
}
String uri = arg_uri;
String fullPath = CFilePath::join(m_root, uri);
#ifndef RHO_NO_RUBY_API
if (rho_ruby_is_started())
{
Route route;
if (dispatch(uri, route)) {
if (verbose) RAWTRACE1("Uri %s is correct route, so enable MVC logic", uri.c_str());
VALUE req = create_request_hash(route.application, route.model, route.action, route.id,
method, uri, query, headers, body);
VALUE data = callFramework(req);
String reply(getStringFromValue(data), getStringLenFromValue(data));
rho_ruby_releaseValue(data);
bool isRedirect = String_startsWith(reply, "HTTP/1.1 301") ||
String_startsWith(reply, "HTTP/1.1 302");
if (!send_response(reply, isRedirect))
return false;
if (method == "GET")
rho_rhodesapp_keeplastvisitedurl(uri.c_str());
if ( sync::RhoconnectClientManager::haveRhoconnectClientImpl() ) {
if (!route.id.empty()) {
sync::RhoconnectClientManager::rho_sync_addobjectnotify_bysrcname(route.model.c_str(), route.id.c_str());
}
}
return true;
}
if (isdir(fullPath)) {
if (verbose) RAWTRACE1("Uri %s is directory, redirecting to index", uri.c_str());
String q = query.empty() ? "" : "?" + query;
HeaderList headers;
headers.push_back(Header("Location", CFilePath::join( uri, "index" RHO_ERB_EXT) + q));
send_response(create_response("301 Moved Permanently", headers), true);
return false;
}
if (isindex(uri)) {
if (!isfile(fullPath)) {
if (verbose) RAWLOG_ERROR1("The file %s was not found", fullPath.c_str());
String error = "<!DOCTYPE html><html><font size=\"+4\"><h2>404 Not Found.</h2> The file " + uri + " was not found.</font></html>";
send_response(create_response("404 Not Found",error));
return false;
}
if (verbose) RAWTRACE1("Uri %s is index file, call serveIndex", uri.c_str());
VALUE req = create_request_hash(route.application, route.model, route.action, route.id,
method, uri, query, headers, body);
VALUE data = callServeIndex((char *)fullPath.c_str(), req);
String reply(getStringFromValue(data), getStringLenFromValue(data));
rho_ruby_releaseValue(data);
if (!send_response(reply))
return false;
if (method == "GET")
rho_rhodesapp_keeplastvisitedurl(uri.c_str());
return true;
}
}
#endif
// Try to send requested file
if (verbose) RAWTRACE1("Uri %s should be regular file, trying to send it", uri.c_str());
PROF_START("READ_FILE");
bool bRes = send_file(uri, headers);
PROF_STOP("READ_FILE");
return bRes;
}
int main(int argc, char **argv)
{
if (argc != 4)
{
fputs("We need 3 arguments\n", stdout);
return 1;
}
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_world_size);
rebuild_target = atoi(argv[1]);
const char *store_dir = argv[2];
const char *data_file = argv[3];
int ntargets = mpi_world_size - 1;
if (ntargets > MAX_STORAGE_TARGETS)
return 1;
if (rebuild_target < 0 || rebuild_target > ntargets)
return 1;
helper = 1;
while (helper == rebuild_target)
helper += 1;
if (helper == rebuild_target)
return 1;
PROF_START(total);
PROF_START(init);
int last_run_fd = -1;
RunData last_run;
memset(&last_run, 0, sizeof(RunData));
if (mpi_rank == 0) {
last_run_fd = open(data_file, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
read(last_run_fd, &last_run, sizeof(RunData));
}
/* Create mapping from storage targets to ranks, and vice versa */
Target targetIDs[MAX_STORAGE_TARGETS] = {{0,0}};
Target targetID = {0,0};
if (mpi_rank != 0)
{
int store_fd = open(store_dir, O_DIRECTORY | O_RDONLY);
int target_ID_fd = openat(store_fd, "targetNumID", O_RDONLY);
char targetID_s[20] = {0};
read(target_ID_fd, targetID_s, sizeof(targetID_s));
close(target_ID_fd);
close(store_fd);
targetID.id = atoi(targetID_s);
targetID.rank = mpi_rank;
}
MPI_Gather(
&targetID, sizeof(Target), MPI_BYTE,
targetIDs, sizeof(Target), MPI_BYTE,
0,
MPI_COMM_WORLD);
if (mpi_rank == 0) {
if (last_run.ntargets != ntargets) {
/* ERROR - new number of targets */
assert(0);
}
for (int i = 0; i < ntargets; i++)
targetIDs[i] = targetIDs[i+1];
for (int i = 0; i < ntargets; i++)
last_run.targetIDs[i].rank = -1;
for (int i = 0; i < ntargets; i++) {
Target target = targetIDs[i];
int j = 0;
int found = 0;
for (; j < ntargets; j++)
if (last_run.targetIDs[j].id == target.id) {
last_run.targetIDs[j] = target;
found = 1;
}
if (!found) {
/* ERROR - new target introduced */
printf(" > %d, %d\n", target.id, target.rank);
assert(0);
}
}
rank2st[0] = -1;
for (int i = 0; i < ntargets; i++)
{
st2rank[i] = last_run.targetIDs[i].rank;
rank2st[st2rank[i]] = i;
}
}
MPI_Bcast(st2rank, sizeof(st2rank), MPI_BYTE, 0, MPI_COMM_WORLD);
MPI_Bcast(rank2st, sizeof(rank2st), MPI_BYTE, 0, MPI_COMM_WORLD);
PROF_END(init);
if (mpi_rank == 0)
printf("%d(rank=%d), %d(rank=%d)\n", rebuild_target, st2rank[rebuild_target], helper, st2rank[helper]);
PROF_START(main_work);
memset(&pr_sender, 0, sizeof(pr_sender));
if (mpi_rank != 0 && rank2st[mpi_rank] != rebuild_target)
{
PersistentDB *pdb = pdb_init();
pdb_iterate(pdb, do_file);
pdb_term(pdb);
if (rank2st[mpi_rank] == helper) {
int dummy;
MPI_Ssend((void*)&dummy, sizeof(dummy), MPI_BYTE, st2rank[rebuild_target], 0, MPI_COMM_WORLD);
}
pr_add_tmp_to_total(&pr_sample);
pr_report_progress(&pr_sender, pr_sample);
pr_report_done(&pr_sender);
}
else if (rank2st[mpi_rank] == rebuild_target)
{
int helper_rank = st2rank[helper];
MPI_Status stat;
int count;
FileInfo fi;
MPI_Recv(&fi, sizeof(FileInfo), MPI_BYTE, helper_rank, 0, MPI_COMM_WORLD, &stat);
MPI_Get_count(&stat, MPI_BYTE, &count);
while (count == sizeof(FileInfo)) {
char key[200];
MPI_Recv(key, sizeof(key), MPI_BYTE, helper_rank, 0, MPI_COMM_WORLD, &stat);
int keylen;
MPI_Get_count(&stat, MPI_BYTE, &keylen);
key[keylen] = '\0';
do_file(key, keylen, &fi);
MPI_Recv(&fi, sizeof(FileInfo), MPI_BYTE, helper_rank, 0, MPI_COMM_WORLD, &stat);
MPI_Get_count(&stat, MPI_BYTE, &count);
}
pr_add_tmp_to_total(&pr_sample);
pr_report_progress(&pr_sender, pr_sample);
pr_report_done(&pr_sender);
}
else if (mpi_rank == 0)
{
printf("st - total files | data read | data written | disk I/O\n");
pr_receive_loop(ntargets-1);
}
PROF_END(main_work);
PROF_END(total);
if (mpi_rank == 0) {
printf("Overall timings: \n");
printf("init | %9.2f ms\n", 1e3*PROF_VAL(init));
printf("main_work | %9.2f ms\n", 1e3*PROF_VAL(main_work));
printf("total | %9.2f ms\n", 1e3*PROF_VAL(total));
}
MPI_Barrier(MPI_COMM_WORLD);
char *iter = hs.corrupt;
for (size_t i = 0; i < hs.corrupt_count; i++)
{
printf("Potentially corrupt chunk: '%s'\n", iter);
iter += strlen(iter);
}
MPI_Finalize();
}
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
jlong dictOffset, jlong dictSize) {
PROF_OPEN;
PROF_START(66);
const jsize sourceDirUtf8Length = env->GetStringUTFLength(sourceDir);
if (sourceDirUtf8Length <= 0) {
AKLOGE("DICT: Can't get sourceDir string");
return 0;
}
char sourceDirChars[sourceDirUtf8Length + 1];
env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
sourceDirChars[sourceDirUtf8Length] = '\0';
int fd = 0;
void *dictBuf = 0;
int adjust = 0;
#ifdef USE_MMAP_FOR_DICTIONARY
/* mmap version */
fd = open(sourceDirChars, O_RDONLY);
if (fd < 0) {
AKLOGE("DICT: Can't open sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno);
return 0;
}
int pagesize = getpagesize();
adjust = static_cast<int>(dictOffset) % pagesize;
int adjDictOffset = static_cast<int>(dictOffset) - adjust;
int adjDictSize = static_cast<int>(dictSize) + adjust;
dictBuf = mmap(0, adjDictSize, PROT_READ, MAP_PRIVATE, fd, adjDictOffset);
if (dictBuf == MAP_FAILED) {
AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);
return 0;
}
dictBuf = static_cast<char *>(dictBuf) + adjust;
#else // USE_MMAP_FOR_DICTIONARY
/* malloc version */
FILE *file = 0;
file = fopen(sourceDirChars, "rb");
if (file == 0) {
AKLOGE("DICT: Can't fopen sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno);
return 0;
}
dictBuf = malloc(dictSize);
if (!dictBuf) {
AKLOGE("DICT: Can't allocate memory region for dictionary. errno=%d", errno);
return 0;
}
int ret = fseek(file, static_cast<long>(dictOffset), SEEK_SET);
if (ret != 0) {
AKLOGE("DICT: Failure in fseek. ret=%d errno=%d", ret, errno);
return 0;
}
ret = fread(dictBuf, dictSize, 1, file);
if (ret != 1) {
AKLOGE("DICT: Failure in fread. ret=%d errno=%d", ret, errno);
return 0;
}
ret = fclose(file);
if (ret != 0) {
AKLOGE("DICT: Failure in fclose. ret=%d errno=%d", ret, errno);
return 0;
}
#endif // USE_MMAP_FOR_DICTIONARY
if (!dictBuf) {
AKLOGE("DICT: dictBuf is null");
return 0;
}
Dictionary *dictionary = 0;
if (BinaryFormat::UNKNOWN_FORMAT
== BinaryFormat::detectFormat(static_cast<uint8_t *>(dictBuf),
static_cast<int>(dictSize))) {
AKLOGE("DICT: dictionary format is unknown, bad magic number");
#ifdef USE_MMAP_FOR_DICTIONARY
releaseDictBuf(static_cast<const char *>(dictBuf) - adjust, adjDictSize, fd);
#else // USE_MMAP_FOR_DICTIONARY
releaseDictBuf(dictBuf, 0, 0);
#endif // USE_MMAP_FOR_DICTIONARY
} else {
dictionary = new Dictionary(dictBuf, static_cast<int>(dictSize), fd, adjust);
}
PROF_END(66);
PROF_CLOSE;
return reinterpret_cast<jlong>(dictionary);
}
