//
// Register a memory block for deferred release.
//
void MachServer::releaseWhenDone(Allocator &alloc, void *memory)
{
if (memory) {
set<Allocation> &releaseSet = perThread().deferredAllocations;
assert(releaseSet.find(Allocation(memory, alloc)) == releaseSet.end());
secinfo("machserver", "allocing register %p with alloc %p", memory, &alloc);
releaseSet.insert(Allocation(memory, alloc));
}
}
void MachRunLoopServer::run(mach_msg_size_t bufferSize, mach_msg_options_t options)
{
// allocate reply buffer
mReplyMessage.setBuffer(bufferSize);
// enable reception
CFAutoPort::enable();
// we are it!
perThread().server = this;
}
//
// Run through the accumulated deferred allocations and release them.
// This is done automatically on every pass through the server loop;
// it must be called by subclasses that implement their loop in some
// other way.
// @@@X Needs to be thread local
//
void MachServer::releaseDeferredAllocations()
{
set<Allocation> &releaseSet = perThread().deferredAllocations;
for (set<Allocation>::iterator it = releaseSet.begin(); it != releaseSet.end(); it++) {
secinfo("machserver", "releasing alloc at %p with %p", it->addr, it->allocator);
// before we release the deferred allocation, zap it so that secrets aren't left in memory
size_t memSize = malloc_size(it->addr);
bzero(it->addr, memSize);
it->allocator->free(it->addr);
}
releaseSet.erase(releaseSet.begin(), releaseSet.end());
}
void DocInverterPerField::processFields(Collection<FieldablePtr> fields, int32_t count)
{
fieldState->reset(docState->doc->getBoost());
int32_t maxFieldLength = docState->maxFieldLength;
bool doInvert = consumer->start(fields, count);
DocumentsWriterPtr docWriter(docState->_docWriter);
DocInverterPerThreadPtr perThread(_perThread);
for (int32_t i = 0; i < count; ++i)
{
FieldablePtr field = fields[i];
if (field->isIndexed() && doInvert)
{
bool anyToken;
if (fieldState->length > 0)
fieldState->position += docState->analyzer->getPositionIncrementGap(fieldInfo->name);
if (!field->isTokenized())
{
// un-tokenized field
String stringValue(field->stringValue());
int32_t valueLength = (int32_t)stringValue.length();
perThread->singleToken->reinit(stringValue, 0, valueLength);
fieldState->attributeSource = perThread->singleToken;
consumer->start(field);
bool success = false;
LuceneException finally;
try
{
consumer->add();
success = true;
}
catch (LuceneException& e)
{
finally = e;
}
if (!success)
docWriter->setAborting();
finally.throwException();
fieldState->offset += valueLength;
++fieldState->length;
++fieldState->position;
anyToken = (valueLength > 0);
}
else
{
// tokenized field
TokenStreamPtr stream;
TokenStreamPtr streamValue(field->tokenStreamValue());
if (streamValue)
stream = streamValue;
else
{
// the field does not have a TokenStream, so we have to obtain one from the analyzer
ReaderPtr reader; // find or make Reader
ReaderPtr readerValue(field->readerValue());
if (readerValue)
reader = readerValue;
else
{
String stringValue(field->stringValue());
perThread->stringReader->init(stringValue);
reader = perThread->stringReader;
}
// Tokenize field and add to postingTable
stream = docState->analyzer->reusableTokenStream(fieldInfo->name, reader);
}
// reset the TokenStream to the first token
stream->reset();
int32_t startLength = fieldState->length;
LuceneException finally;
try
{
int32_t offsetEnd = fieldState->offset - 1;
bool hasMoreTokens = stream->incrementToken();
fieldState->attributeSource = stream;
OffsetAttributePtr offsetAttribute(fieldState->attributeSource->addAttribute<OffsetAttribute>());
PositionIncrementAttributePtr posIncrAttribute(fieldState->attributeSource->addAttribute<PositionIncrementAttribute>());
consumer->start(field);
while (true)
{
// If we hit an exception in stream.next below (which is fairly common, eg if analyzer
// chokes on a given document), then it's non-aborting and (above) this one document
// will be marked as deleted, but still consume a docID
if (!hasMoreTokens)
break;
int32_t posIncr = posIncrAttribute->getPositionIncrement();
fieldState->position += posIncr;
if (fieldState->position > 0)
--fieldState->position;
if (posIncr == 0)
++fieldState->numOverlap;
bool success = false;
try
{
// If we hit an exception in here, we abort all buffered documents since the last
// flush, on the likelihood that the internal state of the consumer is now corrupt
// and should not be flushed to a new segment
consumer->add();
success = true;
}
catch (LuceneException& e)
{
finally = e;
}
if (!success)
docWriter->setAborting();
finally.throwException();
++fieldState->position;
offsetEnd = fieldState->offset + offsetAttribute->endOffset();
if (++fieldState->length >= maxFieldLength)
{
if (docState->infoStream)
*docState->infoStream << L"maxFieldLength " << StringUtils::toString(maxFieldLength) << L" reached for field " << fieldInfo->name << L", ignoring following tokens\n";
break;
}
hasMoreTokens = stream->incrementToken();
}
// trigger streams to perform end-of-stream operations
stream->end();
fieldState->offset += offsetAttribute->endOffset();
anyToken = (fieldState->length > startLength);
}
catch (LuceneException& e)
{
finally = e;
}
stream->close();
finally.throwException();
}
if (anyToken)
fieldState->offset += docState->analyzer->getOffsetGap(field);
fieldState->boost *= field->getBoost();
}
// don't hang onto the field
fields[i].reset();
}
consumer->finish();
endConsumer->finish();
}
void DocInverterPerField::initialize()
{
DocInverterPerThreadPtr perThread(_perThread);
consumer = perThread->consumer->addField(shared_from_this(), fieldInfo);
endConsumer = perThread->endConsumer->addField(shared_from_this(), fieldInfo);
}
void MachServer::runServerThread(bool doTimeout)
{
// allocate request/reply buffers
Message bufRequest(mMaxSize);
Message bufReply(mMaxSize);
// all exits from runServerThread are through exceptions
try {
// register as a worker thread
perThread().server = this;
for (;;) {
// progress hook
eventDone();
// process all pending timers
while (processTimer()) {}
// check for worker idle timeout
{ StLock<Mutex> _(managerLock);
// record idle thread low-water mark in scan interval
if (idleCount < leastIdleWorkers)
leastIdleWorkers = idleCount;
// perform self-timeout processing
if (doTimeout) {
if (workerCount > maxWorkerCount) // someone reduced maxWorkerCount recently...
break; // ... so release this thread immediately
Time::Absolute rightNow = Time::now();
if (rightNow >= nextCheckTime) { // reaping period complete; process
UInt32 idlers = leastIdleWorkers;
secinfo("machserver", "reaping workers: %d %d", (uint32_t) workerCount, (uint32_t) idlers);
nextCheckTime = rightNow + workerTimeout;
leastIdleWorkers = INT_MAX;
if (idlers > 1) // multiple idle threads throughout measuring interval...
break; // ... so release this thread now
}
}
}
// determine next timeout (if any)
bool indefinite = false;
Time::Interval timeout = workerTimeout;
{ StLock<Mutex> _(managerLock);
if (timers.empty()) {
indefinite = !doTimeout;
} else {
timeout = max(Time::Interval(0), timers.next() - Time::now());
if (doTimeout && workerTimeout < timeout)
timeout = workerTimeout;
}
}
// receive next IPC request (or wait for timeout)
mach_msg_return_t mr = indefinite ?
mach_msg_overwrite(bufRequest,
MACH_RCV_MSG | mMsgOptions,
0, mMaxSize, mPortSet,
MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL,
(mach_msg_header_t *) 0, 0)
:
mach_msg_overwrite(bufRequest,
MACH_RCV_MSG | MACH_RCV_TIMEOUT | MACH_RCV_INTERRUPT | mMsgOptions,
0, mMaxSize, mPortSet,
mach_msg_timeout_t(timeout.mSeconds()), MACH_PORT_NULL,
(mach_msg_header_t *) 0, 0);
switch (mr) {
case MACH_MSG_SUCCESS:
// process received request message below
break;
default:
secinfo("machserver", "received error: %d", mr);
continue;
}
// process received message
if (bufRequest.msgId() >= MACH_NOTIFY_FIRST &&
bufRequest.msgId() <= MACH_NOTIFY_LAST) {
// mach kernel notification message
// we assume this is quick, so no thread arbitration here
cdsa_notify_server(bufRequest, bufReply);
} else {
// normal request message
StLock<MachServer, &MachServer::busy, &MachServer::idle> _(*this);
secinfo("machserver", "begin request: %d, %d", bufRequest.localPort().port(), bufRequest.msgId());
// try subsidiary handlers first
bool handled = false;
for (HandlerSet::const_iterator it = mHandlers.begin();
it != mHandlers.end(); it++)
if (bufRequest.localPort() == (*it)->port()) {
(*it)->handle(bufRequest, bufReply);
handled = true;
}
if (!handled) {
// unclaimed, send to main handler
handle(bufRequest, bufReply);
}
secinfo("machserver", "end request");
}
// process reply generated by handler
if (!(bufReply.bits() & MACH_MSGH_BITS_COMPLEX) &&
bufReply.returnCode() != KERN_SUCCESS) {
if (bufReply.returnCode() == MIG_NO_REPLY)
continue;
// don't destroy the reply port right, so we can send an error message
bufRequest.remotePort(MACH_PORT_NULL);
mach_msg_destroy(bufRequest);
}
if (bufReply.remotePort() == MACH_PORT_NULL) {
// no reply port, so destroy the reply
if (bufReply.bits() & MACH_MSGH_BITS_COMPLEX)
bufReply.destroy();
continue;
}
/*
* We don't want to block indefinitely because the client
* isn't receiving messages from the reply port.
* If we have a send-once right for the reply port, then
* this isn't a concern because the send won't block.
* If we have a send right, we need to use MACH_SEND_TIMEOUT.
* To avoid falling off the kernel's fast RPC path unnecessarily,
* we only supply MACH_SEND_TIMEOUT when absolutely necessary.
*/
mr = mach_msg_overwrite(bufReply,
(MACH_MSGH_BITS_REMOTE(bufReply.bits()) ==
MACH_MSG_TYPE_MOVE_SEND_ONCE) ?
MACH_SEND_MSG | mMsgOptions :
MACH_SEND_MSG | MACH_SEND_TIMEOUT | mMsgOptions,
bufReply.length(), 0, MACH_PORT_NULL,
0, MACH_PORT_NULL, NULL, 0);
switch (mr) {
case MACH_MSG_SUCCESS:
break;
default:
secinfo("machserver", "send error: %d %d", mr, bufReply.remotePort().port());
bufReply.destroy();
break;
}
// clean up after the transaction
releaseDeferredAllocations();
}
perThread().server = NULL;
} catch (...) {
perThread().server = NULL;
throw;
}
}
MachRunLoopServer::~MachRunLoopServer()
{
// no longer active on this thread
perThread().server = NULL;
}
