/**
* For definition of "check point32_t" see IndexWriter comments:
* "Clarification: Check Point32_ts (and commits)".
*
* Writer calls this when it has made a "consistent
* change" to the index, meaning new files are written to
* the index and the in-memory SegmentInfos have been
* modified to point32_t to those files.
*
* This may or may not be a commit (segments_N may or may
* not have been written).
*
* We simply incref the files referenced by the new
* SegmentInfos and decref the files we had previously
* seen (if any).
*
* If this is a commit, we also call the policy to give it
* a chance to remove other commits. If any commits are
* removed, we decref their files as well.
*/
void IndexFileDeleter::checkpoint(SegmentInfos* segmentInfos, bool isCommit) {
if (infoStream != NULL) {
message(string("now checkpoint \"") + segmentInfos->getCurrentSegmentFileName() + "\" [" +
Misc::toString(segmentInfos->size()) + " segments ; isCommit = " + Misc::toString(isCommit) + "]");
}
// Try again now to delete any previously un-deletable
// files (because they were in use, on Windows):
deletePendingFiles();
// Incref the files:
incRef(segmentInfos, isCommit);
const vector<string>* docWriterFiles = NULL;
if (docWriter != NULL) {
docWriterFiles = &docWriter->files();
if (!docWriterFiles->empty())
incRef(*docWriterFiles);
else
docWriterFiles = NULL;
}
if (isCommit) {
// Append to our commits list:
commits.push_back(_CLNEW CommitPoint(this, segmentInfos));
// Tell policy so it can remove commits:
policy->onCommit(commits);
// Decref files for commits that were deleted by the policy:
deleteCommits();
}
// DecRef old files from the last checkpoint, if any:
int32_t size = lastFiles.size();
if (size > 0) {
for(int32_t i=0;i<size;i++)
decRef(lastFiles[i]);
lastFiles.clear();
}
if (!isCommit) {
// Save files so we can decr on next checkpoint/commit:
size = segmentInfos->size();
for(int32_t i=0;i<size;i++) {
SegmentInfo* segmentInfo = segmentInfos->info(i);
if (segmentInfo->dir == directory) {
const vector<string>& files = segmentInfo->files();
lastFiles.insert(lastFiles.end(), files.begin(), files.end());
}
}
}
if (docWriterFiles != NULL)
lastFiles.insert(lastFiles.end(), docWriterFiles->begin(),docWriterFiles->end());
}
SharedHandle::SharedHandle(const SharedHandle& rhs):
m_handle(rhs.m_handle),
m_serverHandle(rhs.m_serverHandle),
m_pSharedCount(rhs.m_pSharedCount)
{
incRef();
}
void work_fail(Job *job)
{
MemBlock *block = new_response(MSG_WORK_FAIL, strlen(job->handle), (unsigned char*)job->handle);
incRef(block); // Make sure the first listener doesn't decRef to 0 causing the block to be returned
g_ptr_array_foreach(job->listeners, (GFunc)client_send, block);
decRef(block);
remove_job(job);
}
SharedHandle& SharedHandle::operator =(const SharedHandle& rhs)
{
decRef();
m_handle = rhs.m_handle;
m_serverHandle = rhs.m_serverHandle;
m_pSharedCount = rhs.m_pSharedCount;
incRef();
return *this;
}
void Context::cleanup() {
incRef();
clear();
std::vector<Context*> subs;
std::swap(mSubs, subs);
for (size_t i=0; i<subs.size(); ++i) {
subs[i]->cleanup();
}
decRef();
}
template <> void RangeType<int>::set(Value *v, int start,int end,int step){
v->clr();
v->t = this;
Range<int> *r = new Range<int>();
r->start = start;
r->end = end;
r->step = step;
v->v.irange = r;
incRef(v);
}
void IndexFileDeleter::incRef(SegmentInfos* segmentInfos, bool isCommit) {
int32_t size = segmentInfos->size();
for(int32_t i=0;i<size;i++) {
SegmentInfo* segmentInfo = segmentInfos->info(i);
if (segmentInfo->dir == directory) {
incRef(segmentInfo->files());
}
}
if (isCommit) {
// Since this is a commit point32_t, also incref its
// segments_N file:
getRefCount(segmentInfos->getCurrentSegmentFileName().c_str())->IncRef();
}
}
// returns true if we're the only copy.
bool getWriteLock()
{
if (m_pRefCount->decAndTest())
{
// only copy--don't need to clone, also not a race condition
incRef();
return true;
}
else
{
// need to become unique
m_pRefCount = new RefCount;
return false;
}
}
String::String(const String& other) :
_data(other._data)
{
incRef();
}
IndexFileDeleter::IndexFileDeleter(Directory* directory, IndexDeletionPolicy* policy,
SegmentInfos* segmentInfos, std::ostream* infoStream, DocumentsWriter* docWriter):
refCounts( RefCountsType(true,true) ), commits(CommitsType(true))
{
this->docWriter = docWriter;
this->infoStream = infoStream;
if (infoStream != NULL)
message( string("init: current segments file is \"") + segmentInfos->getCurrentSegmentFileName() + "\"; deletionPolicy=" + policy->getObjectName());
this->policy = policy;
this->directory = directory;
CommitPoint* currentCommitPoint = NULL;
// First pass: walk the files and initialize our ref
// counts:
int64_t currentGen = segmentInfos->getGeneration();
const IndexFileNameFilter* filter = IndexFileNameFilter::getFilter();
vector<string> files;
if ( !directory->list(&files) )
_CLTHROWA(CL_ERR_IO, (string("cannot read directory ") + directory->toString() + ": list() returned NULL").c_str());
for(size_t i=0;i<files.size();i++) {
string& fileName = files.at(i);
if (filter->accept(NULL, fileName.c_str()) && fileName.compare(IndexFileNames::SEGMENTS_GEN) != 0) {
// Add this file to refCounts with initial count 0:
getRefCount(fileName.c_str());
if ( strncmp(fileName.c_str(), IndexFileNames::SEGMENTS, strlen(IndexFileNames::SEGMENTS)) == 0 ) {
// This is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (SegmentInfos::generationFromSegmentsFileName(fileName.c_str()) <= currentGen) {
if (infoStream != NULL) {
message("init: load commit \"" + fileName + "\"");
}
SegmentInfos sis;
bool failed = false;
try {
sis.read(directory, fileName.c_str());
} catch (CLuceneError& e) {
if ( e.number() != CL_ERR_IO ){
throw e;
}
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream != NULL) {
message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point32_t");
}
failed = true;
}
if (!failed) {
CommitPoint* commitPoint = _CLNEW CommitPoint(this,&sis);
if (sis.getGeneration() == segmentInfos->getGeneration()) {
currentCommitPoint = commitPoint;
}
commits.push_back(commitPoint);
incRef(&sis, true);
}
}
}
}
}
if (currentCommitPoint == NULL) {
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. This can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point32_t:
SegmentInfos sis;
try {
sis.read(directory, segmentInfos->getCurrentSegmentFileName().c_str());
} catch (CLuceneError& e) {
if ( e.number() == CL_ERR_IO ){
_CLTHROWA(CL_ERR_CorruptIndex, "failed to locate current segments_N file");
}
}
if (infoStream != NULL)
message("forced open of current segments file " + segmentInfos->getCurrentSegmentFileName());
currentCommitPoint = _CLNEW CommitPoint(this,&sis);
commits.push_back(currentCommitPoint);
incRef(&sis, true);
}
// We keep commits list in sorted order (oldest to newest):
std::sort(commits.begin(), commits.end(), CommitPoint::sort);
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
RefCountsType::iterator it = refCounts.begin();
while(it != refCounts.end()) {
char* fileName = it->first;
RefCount* rc = it->second;
if (0 == rc->count) {
if (infoStream != NULL) {
message( string("init: removing unreferenced file \"") + fileName + "\"");
}
deleteFile(fileName);
}
it++;
}
// Finally, give policy a chance to remove things on
// startup:
policy->onInit(commits);
// It's OK for the onInit to remove the current commit
// point; we just have to checkpoint our in-memory
// SegmentInfos to protect those files that it uses:
if (currentCommitPoint->deleted) {
checkpoint(segmentInfos, false);
}
deleteCommits();
}
void useRefCountOf(const ReferenceBase& arg)
{
decRef();
m_pRefCount = arg.m_pRefCount;
incRef();
}
/******************************************************************
* This function is called when an event occurs on a client socket
******************************************************************/
void client_cb(int fd, short events, void *arg)
{
assert(arg != NULL);
Client *cli = arg;
int free = 0;
// g_hash_table_foreach(g_jobqueue, _print_queue, NULL);
if ((events & EV_WRITE) != 0) {
event_del(&cli->evt);
cli->evt.ev_events = EV_READ|EV_PERSIST;
event_add(&cli->evt, NULL);
if (client_flush(cli) < 0) {
free = 1;
}
}
if ((events & EV_READ) != 0) {
int ret = 0;
if (!cli->buffer_in) {
cli->buffer_in = getBlock(HEADER_SIZE);
incRef(cli->buffer_in);
ret = client_recv(cli, HEADER_SIZE);
}
if (ret >= 0) {
/* Make sure we don't over-read into the next packet */
int psize = HEADER_SIZE;
if (cli->buffer_in->nbytes >= HEADER_SIZE) {
if (ntohl(*(uint32_t*)(cli->buffer_in->bytes + HEADER_OFFSET_MAGIC)) != MAGIC_REQUEST) {
free = 1;
g_warning("[%s] Invalid MAGIC", cli->id);
goto free_client;
}
psize = HEADER_SIZE + ntohl(*(uint32_t*)(cli->buffer_in->bytes + HEADER_OFFSET_SIZE));
/* If the input block isn't large enough to receive the
entire packet then switch to one that is */
if (psize > cli->buffer_in->size) {
#if DEBUG
g_debug("Switching to bigger block (pktsize=%d)", psize);
#endif
/* Create new (bigger) block */
MemBlock *block = getBlock(psize + 1); /* +1 for terminating NULL to make args easier to work with */
if (!block) {
g_error("Failed to get block of size %d", psize);
free = 1;
goto free_client;
}
incRef(block);
/* Copy bytes into new block */
block->nbytes = cli->buffer_in->nbytes;
memmove(block->bytes, cli->buffer_in->bytes, cli->buffer_in->nbytes);
/* Swap blocks */
decRef(cli->buffer_in);
cli->buffer_in = block;
}
}
int num = psize - cli->buffer_in->nbytes;
if (num > 0)
ret = client_recv(cli, num);
}
if (ret < 0) {
#if DEBUG
g_debug("[%s] Connection on closed", cli->id);
#endif
free = 1;
} else if (ret >= 0) {
if (process_client(cli) != 0) {
g_warning("[%s] Processing of client failed", cli->id);
free = 1;
}
}
}
/*if ((events & (EV_READ|EV_WRITE)) == 0) {
g_warning("[%s] unhandled event %d", __func__, events);
}*/
free_client:
if (free != 0) {
#if DEBUG
g_message("[%s] Client disconnected", cli->id);
#endif
/*printf("[%s] Removing client %d\n", __func__, cli->fd);*/
close(cli->fd);
cli->fd = -1;
fail_working_jobs(cli);
stop_all_listening(cli);
unregister_all_abilities(cli);
event_del(&cli->evt);
g_ptr_array_remove_fast(g_clients, cli);
client_free(cli);
}
}
/**
* @brief Constructor with defined initialized dict size.
*/
sidexDict::sidexDict(long tableSize)
{
m_hashTable = new sidexHashTable(tableSize);
incRef();
}
/**
* @brief Constructor. The initial dict size is 100.
*/
sidexDict::sidexDict()
{
m_hashTable = new sidexHashTable();
incRef();
}
NODE* parse(char **exp) {
if (!sym_map) parser_init();
debug("Parse List: %s\n",*exp);
NODE *head = NIL;
while (**exp) {
switch (*((*exp)++)) {
case '\'': {
debug("to quote: %s\n",*exp);
NODE *quoted = parse(exp);
debugVal(quoted->data,"quoted: ");
list_push(newNODE(newPRIMFUNC(SPEC_QUOTE,l_quote),newNODE(quoted->data,NIL)),&head);
if (quoted->addr) head = list_join(list_reverse((NODE*)quoted->addr),head);
head = list_reverse(head);
debugVal(head,"expression: ");
return head;
}
case '(':
list_push(parse(exp),&head);
break;
case ')':
head = list_reverse(head);
debugVal(head,"expression: ");
return head;
case ';':
while (**exp != '\0' && **exp != '\r' && **exp != '\n') (*exp)++;
break;
case '\n':
case '\r':
case '\t':
case ' ':
break;
default: {
char *sym = *exp-1;
debug("origin: %s\n",sym);
while (**exp && **exp != ' ' && **exp != ')' && **exp != '\n' && **exp != '\r' && **exp != '\t') (*exp)++;
char old = **exp;
**exp = 0;
debug("literal: %s\n",sym);
switch (sym[0]) {
case '+':
case '-':
if (!isdigit(sym[1])) {
list_push(newSYMBOL(intern(sym)),&head);
break;
}
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
{
bool real = false;
char *scn = sym+1;
while (*scn) {
if (*scn == '.') {
real = true;
} else if (!isdigit(*scn)) {
error("Malformed number character %c",*scn);
}
scn++;
}
if (real) {
list_push(newREAL(atof(sym)),&head);
} else {
list_push(newINTEGER(atoi(sym)),&head);
}
}
break;
default:
list_push(newSYMBOL(intern(sym)),&head);
break;
}
**exp = old;
if (head->data->type == ID_SYMBOL) {
NODE *literal;
if ((literal = binmap_find(head->data,literal_map))) {
NODE *last = (NODE*)head->addr;
incRef(last);
decRef(head);
incRef(literal->addr);
head = last;
list_push(literal->addr,&head);
decRef(literal);
}
}
debugVal(head,"parsed: ");
}
}
}
head = list_reverse(head);
debugVal(head,"dangling: ");
return head;
}
