void TapFile::_pad(google::protobuf::io::FileOutputStream* fos, int remnant) {
int padBytes = Alignment - 1 - ((fos->ByteCount() + Alignment-1) % Alignment);
padBytes -= remnant;
char pad[Alignment];
memset(pad, 0xFA, padBytes);
_raw(fos, pad, padBytes);
}
EQRawApplicationPacket *EQStream::MakeApplicationPacket(EQProtocolPacket *p)
{
EQRawApplicationPacket *ap=NULL;
_log(NET__APP_CREATE, _L "Creating new application packet, length %d" __L, p->size);
_raw(NET__APP_CREATE_HEX, 0xFFFF, p);
ap = p->MakeAppPacket();
return ap;
}
void EQStream::ProcessPacket(EQProtocolPacket *p)
{
uint32 processed=0,subpacket_length=0;
if (p == NULL)
return;
// Raw Application packet
if (p->opcode > 0xff) {
p->opcode = htons(p->opcode); //byte order is backwards in the protocol packet
EQRawApplicationPacket *ap=MakeApplicationPacket(p);
if (ap)
InboundQueuePush(ap);
return;
}
if (!Session && p->opcode!=OP_SessionRequest && p->opcode!=OP_SessionResponse) {
_log(NET__DEBUG, _L "Session not initialized, packet ignored" __L);
_raw(NET__DEBUG, 0xFFFF, p);
return;
}
switch (p->opcode) {
case OP_Combined: {
processed=0;
while(processed < p->size) {
subpacket_length=*(p->pBuffer+processed);
EQProtocolPacket *subp=MakeProtocolPacket(p->pBuffer+processed+1,subpacket_length);
_log(NET__NET_CREATE, _L "Extracting combined packet of length %d" __L, subpacket_length);
_raw(NET__NET_CREATE_HEX, 0xFFFF, subp);
subp->copyInfo(p);
ProcessPacket(subp);
delete subp;
processed+=subpacket_length+1;
}
}
break;
case OP_AppCombined: {
processed=0;
while(processed<p->size) {
EQRawApplicationPacket *ap=NULL;
if ((subpacket_length=(unsigned char)*(p->pBuffer+processed))!=0xff) {
_log(NET__NET_CREATE, _L "Extracting combined app packet of length %d, short len" __L, subpacket_length);
ap=MakeApplicationPacket(p->pBuffer+processed+1,subpacket_length);
processed+=subpacket_length+1;
} else {
subpacket_length=ntohs(*(uint16 *)(p->pBuffer+processed+1));
_log(NET__NET_CREATE, _L "Extracting combined app packet of length %d, short len" __L, subpacket_length);
ap=MakeApplicationPacket(p->pBuffer+processed+3,subpacket_length);
processed+=subpacket_length+3;
}
if (ap) {
ap->copyInfo(p);
InboundQueuePush(ap);
}
}
}
break;
case OP_Packet: {
if(!p->pBuffer || (p->Size() < 4))
{
_log(NET__ERROR, _L "Received OP_Packet that was of malformed size" __L);
break;
}
uint16 seq=ntohs(*(uint16 *)(p->pBuffer));
SeqOrder check=CompareSequence(NextInSeq,seq);
if (check == SeqFuture) {
_log(NET__DEBUG, _L "Future OP_Packet: Expecting Seq=%d, but got Seq=%d" __L, NextInSeq, seq);
_raw(NET__DEBUG, seq, p);
PacketQueue[seq]=p->Copy();
_log(NET__APP_TRACE, _L "OP_Packet Queue size=%d" __L, PacketQueue.size());
//SendOutOfOrderAck(seq);
} else if (check == SeqPast) {
_log(NET__DEBUG, _L "Duplicate OP_Packet: Expecting Seq=%d, but got Seq=%d" __L, NextInSeq, seq);
_raw(NET__DEBUG, seq, p);
SendOutOfOrderAck(seq); //we already got this packet but it was out of order
} else {
// In case we did queue one before as well.
EQProtocolPacket *qp=RemoveQueue(seq);
if (qp) {
_log(NET__NET_TRACE, "OP_Packet: Removing older queued packet with sequence %d", seq);
delete qp;
}
SetNextAckToSend(seq);
NextInSeq++;
// Check for an embedded OP_AppCombinded (protocol level 0x19)
if (*(p->pBuffer+2)==0x00 && *(p->pBuffer+3)==0x19) {
EQProtocolPacket *subp=MakeProtocolPacket(p->pBuffer+2,p->size-2);
_log(NET__NET_CREATE, _L "seq %d, Extracting combined packet of length %d" __L, seq, subp->size);
_raw(NET__NET_CREATE_HEX, seq, subp);
subp->copyInfo(p);
ProcessPacket(subp);
delete subp;
} else {
EQRawApplicationPacket *ap=MakeApplicationPacket(p->pBuffer+2,p->size-2);
if (ap) {
ap->copyInfo(p);
InboundQueuePush(ap);
}
}
}
}
break;
case OP_Fragment: {
if(!p->pBuffer || (p->Size() < 4))
{
_log(NET__ERROR, _L "Received OP_Fragment that was of malformed size" __L);
break;
}
uint16 seq=ntohs(*(uint16 *)(p->pBuffer));
SeqOrder check=CompareSequence(NextInSeq,seq);
if (check == SeqFuture) {
_log(NET__DEBUG, _L "Future OP_Fragment: Expecting Seq=%d, but got Seq=%d" __L, NextInSeq, seq);
_raw(NET__DEBUG, seq, p);
PacketQueue[seq]=p->Copy();
_log(NET__APP_TRACE, _L "OP_Fragment Queue size=%d" __L, PacketQueue.size());
//SendOutOfOrderAck(seq);
} else if (check == SeqPast) {
_log(NET__DEBUG, _L "Duplicate OP_Fragment: Expecting Seq=%d, but got Seq=%d" __L, NextInSeq, seq);
_raw(NET__DEBUG, seq, p);
SendOutOfOrderAck(seq);
} else {
// In case we did queue one before as well.
EQProtocolPacket *qp=RemoveQueue(seq);
if (qp) {
_log(NET__NET_TRACE, "OP_Fragment: Removing older queued packet with sequence %d", seq);
delete qp;
}
SetNextAckToSend(seq);
NextInSeq++;
if (oversize_buffer) {
memcpy(oversize_buffer+oversize_offset,p->pBuffer+2,p->size-2);
oversize_offset+=p->size-2;
_log(NET__NET_TRACE, _L "Fragment of oversized of length %d, seq %d: now at %d/%d" __L, p->size-2, seq, oversize_offset, oversize_length);
if (oversize_offset==oversize_length) {
if (*(p->pBuffer+2)==0x00 && *(p->pBuffer+3)==0x19) {
EQProtocolPacket *subp=MakeProtocolPacket(oversize_buffer,oversize_offset);
_log(NET__NET_CREATE, _L "seq %d, Extracting combined oversize packet of length %d" __L, seq, subp->size);
//_raw(NET__NET_CREATE_HEX, subp);
subp->copyInfo(p);
ProcessPacket(subp);
delete subp;
} else {
EQRawApplicationPacket *ap=MakeApplicationPacket(oversize_buffer,oversize_offset);
_log(NET__NET_CREATE, _L "seq %d, completed combined oversize packet of length %d" __L, seq, ap->size);
if (ap) {
ap->copyInfo(p);
InboundQueuePush(ap);
}
}
delete[] oversize_buffer;
oversize_buffer=NULL;
oversize_offset=0;
}
} else {
oversize_length=ntohl(*(uint32 *)(p->pBuffer+2));
oversize_buffer=new unsigned char[oversize_length];
memcpy(oversize_buffer,p->pBuffer+6,p->size-6);
oversize_offset=p->size-6;
_log(NET__NET_TRACE, _L "First fragment of oversized of seq %d: now at %d/%d" __L, seq, oversize_offset, oversize_length);
}
}
}
break;
case OP_KeepAlive: {
#ifndef COLLECTOR
NonSequencedPush(new EQProtocolPacket(p->opcode,p->pBuffer,p->size));
_log(NET__NET_TRACE, _L "Received and queued reply to keep alive" __L);
#endif
}
break;
case OP_Ack: {
if(!p->pBuffer || (p->Size() < 4))
{
_log(NET__ERROR, _L "Received OP_Ack that was of malformed size" __L);
break;
}
#ifndef COLLECTOR
uint16 seq=ntohs(*(uint16 *)(p->pBuffer));
AckPackets(seq);
#ifdef RETRANSMITS
retransmittimer = Timer::GetCurrentTime();
#endif
#endif
}
break;
case OP_SessionRequest: {
if(p->Size() < sizeof(SessionRequest))
{
_log(NET__ERROR, _L "Received OP_SessionRequest that was of malformed size" __L);
break;
}
#ifndef COLLECTOR
if (GetState()==ESTABLISHED) {
_log(NET__ERROR, _L "Received OP_SessionRequest in ESTABLISHED state (%d)" __L, GetState());
/*RemoveData();
init();
State=UNESTABLISHED;*/
_SendDisconnect();
SetState(CLOSED);
break;
}
#endif
//cout << "Got OP_SessionRequest" << endl;
init();
OutboundQueueClear();
SessionRequest *Request=(SessionRequest *)p->pBuffer;
Session=ntohl(Request->Session);
SetMaxLen(ntohl(Request->MaxLength));
_log(NET__NET_TRACE, _L "Received OP_SessionRequest: session %lu, maxlen %d" __L, (unsigned long)Session, MaxLen);
SetState(ESTABLISHED);
#ifndef COLLECTOR
Key=0x11223344;
SendSessionResponse();
#endif
}
break;
case OP_SessionResponse: {
if(p->Size() < sizeof(SessionResponse))
{
_log(NET__ERROR, _L "Received OP_SessionResponse that was of malformed size" __L);
break;
}
init();
OutboundQueueClear();
SessionResponse *Response=(SessionResponse *)p->pBuffer;
SetMaxLen(ntohl(Response->MaxLength));
Key=ntohl(Response->Key);
NextInSeq=0;
SetState(ESTABLISHED);
if (!Session)
Session=ntohl(Response->Session);
compressed=(Response->Format&FLAG_COMPRESSED);
encoded=(Response->Format&FLAG_ENCODED);
_log(NET__NET_TRACE, _L "Received OP_SessionResponse: session %lu, maxlen %d, key %lu, compressed? %s, encoded? %s" __L, (unsigned long)Session, MaxLen, (unsigned long)Key, compressed?"yes":"no", encoded?"yes":"no");
// Kinda kludgy, but trie for now
if (StreamType==UnknownStream) {
if (compressed) {
if (remote_port==9000 || (remote_port==0 && p->src_port==9000)) {
SetStreamType(WorldStream);
} else {
SetStreamType(ZoneStream);
}
} else if (encoded) {
SetStreamType(ChatOrMailStream);
} else {
SetStreamType(LoginStream);
}
}
}
break;
case OP_SessionDisconnect: {
//NextInSeq=0;
EQStreamState state = GetState();
if(state == ESTABLISHED) {
//client initiated disconnect?
_log(NET__NET_TRACE, _L "Received unsolicited OP_SessionDisconnect. Treating like a client-initiated disconnect." __L);
_SendDisconnect();
SetState(CLOSED);
} else if(state == CLOSING) {
//we were waiting for this anyways, ignore pending messages, send the reply and be closed.
_log(NET__NET_TRACE, _L "Received OP_SessionDisconnect when we have a pending close, they beat us to it. Were happy though." __L);
_SendDisconnect();
SetState(CLOSED);
} else {
//we are expecting this (or have already gotten it, but dont care either way)
_log(NET__NET_TRACE, _L "Received expected OP_SessionDisconnect. Moving to closed state." __L);
SetState(CLOSED);
}
}
break;
case OP_OutOfOrderAck: {
if(!p->pBuffer || (p->Size() < 4))
{
_log(NET__ERROR, _L "Received OP_OutOfOrderAck that was of malformed size" __L);
break;
}
#ifndef COLLECTOR
uint16 seq=ntohs(*(uint16 *)(p->pBuffer));
MOutboundQueue.lock();
if(uint16(SequencedBase + SequencedQueue.size()) != NextOutSeq) {
_log(NET__ERROR, _L "Pre-OOA Invalid Sequenced queue: BS %d + SQ %d != NOS %d" __L, SequencedBase, SequencedQueue.size(), NextOutSeq);
}
if(NextSequencedSend > SequencedQueue.size()) {
_log(NET__ERROR, _L "Pre-OOA Next Send Sequence is beyond the end of the queue NSS %d > SQ %d" __L, NextSequencedSend, SequencedQueue.size());
}
//if the packet they got out of order is between our last acked packet and the last sent packet, then its valid.
if (CompareSequence(SequencedBase,seq) != SeqPast && CompareSequence(NextOutSeq,seq) == SeqPast) {
_log(NET__NET_TRACE, _L "Received OP_OutOfOrderAck for sequence %d, starting retransmit at the start of our unacked buffer (seq %d, was %d)." __L,
seq, SequencedBase, SequencedBase+NextSequencedSend);
#ifdef RETRANSMITS
if (!RuleB(EQStream, RetransmitAckedPackets)) {
#endif
uint16 sqsize = SequencedQueue.size();
uint16 index = seq - SequencedBase;
_log(NET__NET_TRACE, _L " OP_OutOfOrderAck marking packet acked in queue (queue index = %d, queue size = %d)." __L, index, sqsize);
if (index < sqsize) {
deque<EQProtocolPacket *>::iterator sitr;
sitr = SequencedQueue.begin();
sitr += index;
(*sitr)->acked = true;
}
#ifdef RETRANSMITS
}
if (RuleR(EQStream, RetransmitTimeoutMult)) { // only choose new behavior if multiplier is set
retransmittimer = Timer::GetCurrentTime();
}
#endif
NextSequencedSend = 0;
} else {
_log(NET__NET_TRACE, _L "Received OP_OutOfOrderAck for out-of-window %d. Window (%d->%d)." __L, seq, SequencedBase, NextOutSeq);
}
if(uint16(SequencedBase + SequencedQueue.size()) != NextOutSeq) {
_log(NET__ERROR, _L "Post-OOA Invalid Sequenced queue: BS %d + SQ %d != NOS %d" __L, SequencedBase, SequencedQueue.size(), NextOutSeq);
}
if(NextSequencedSend > SequencedQueue.size()) {
_log(NET__ERROR, _L "Post-OOA Next Send Sequence is beyond the end of the queue NSS %d > SQ %d" __L, NextSequencedSend, SequencedQueue.size());
}
MOutboundQueue.unlock();
#endif
}
break;
case OP_SessionStatRequest: {
if(p->Size() < sizeof(SessionStats))
{
_log(NET__ERROR, _L "Received OP_SessionStatRequest that was of malformed size" __L);
break;
}
#ifndef COLLECTOR
SessionStats *Stats=(SessionStats *)p->pBuffer;
_log(NET__NET_TRACE, _L "Received Stats: %lu packets received, %lu packets sent, Deltas: local %lu, (%lu <- %lu -> %lu) remote %lu" __L,
(unsigned long)ntohl(Stats->packets_received), (unsigned long)ntohl(Stats->packets_sent), (unsigned long)ntohl(Stats->last_local_delta),
(unsigned long)ntohl(Stats->low_delta), (unsigned long)ntohl(Stats->average_delta),
(unsigned long)ntohl(Stats->high_delta), (unsigned long)ntohl(Stats->last_remote_delta));
uint64 x=Stats->packets_received;
Stats->packets_received=Stats->packets_sent;
Stats->packets_sent=x;
NonSequencedPush(new EQProtocolPacket(OP_SessionStatResponse,p->pBuffer,p->size));
AdjustRates(ntohl(Stats->average_delta));
#ifdef RETRANSMITS
if (RuleR(EQStream, RetransmitTimeoutMult) && ntohl(Stats->average_delta)) {
//recalculate retransmittimeout using the larger of the last rtt or average rtt, which is multiplied by the rule value
if((ntohl(Stats->last_local_delta) + ntohl(Stats->last_remote_delta)) > (ntohl(Stats->average_delta) * 2)) {
retransmittimeout = (ntohl(Stats->last_local_delta) + ntohl(Stats->last_remote_delta)) * RuleR(EQStream, RetransmitTimeoutMult);
} else {
retransmittimeout = ntohl(Stats->average_delta) * 2 * RuleR(EQStream, RetransmitTimeoutMult);
}
if(retransmittimeout > RuleI(EQStream, RetransmitTimeoutMax))
retransmittimeout = RuleI(EQStream, RetransmitTimeoutMax);
_log(NET__NET_TRACE, _L "Retransmit timeout recalculated to %dms" __L, retransmittimeout);
}
#endif
#endif
}
break;
case OP_SessionStatResponse: {
_log(NET__NET_TRACE, _L "Received OP_SessionStatResponse. Ignoring." __L);
}
break;
case OP_OutOfSession: {
_log(NET__NET_TRACE, _L "Received OP_OutOfSession. Ignoring." __L);
}
break;
default:
EQRawApplicationPacket *ap = MakeApplicationPacket(p);
if (ap)
InboundQueuePush(ap);
break;
}
}
QBStatus TapFile::close()
{
static const char *method = "TapFile::close()";
QBStatus status;
if (_fd == 0) return(QB_SUCCESS);
if (_mode != QB_READ) {
// flush out the last data block
_flush();
//
// close out and serialize the whole index
//
// yes making an extra copy of the entire index in a string is a bad idea
// but i had trouble getting to the actual buffer to copy
//
// this should probably actually go into a temporary File, and then get copied
// over with sendFile(). the data would usually all stay in the buffercache
//
// delete the coded output stream to flush its contents to the OstreamOutputStrean
delete _indexOStream;
_indexOStream = NULL;
// delete the OstreamOutputStream to flush its contents to the ostringstream
delete _indexOstreamOStream;
_indexOstreamOStream = NULL;
// extract the data
string s = _indexOstringstream->str();
// now delete the ostringstream to free memory
delete _indexOstringstream;
_indexOstringstream = NULL;
// output the wole index and pad to the next sector boundary
_trailer.set_index_offset(_fileOStream->ByteCount());
_raw(_fileOStream, s.c_str(), s.length());
_pad(_fileOStream, 0);
_trailer.set_index_bytes(_fileOStream->ByteCount() - _trailer.index_offset());
//
// write the trailer including signature at the end of the File
//
uint64_t trailerOffset = _fileOStream->ByteCount();
_oStream = new google::protobuf::io::CodedOutputStream(_fileOStream);
_oStream->WriteRaw("trai",4);
_oStream->WriteRaw("none",4);
_oStream->WriteVarint32(_trailer.ByteSize());
_trailer.SerializeWithCachedSizes(_oStream);
_pad(_oStream, 16); // pad to 16 bytes before the sector end
_oStream->WriteLittleEndian64(trailerOffset);
_oStream->WriteRaw("tapproto",8);
delete _oStream;
_oStream = NULL;
//
// final cleanup of the File
//
_fileOStream->Close();
delete _fileOStream;
_fileOStream = NULL;
}
else {
int retval = ::close(_fd);
if (retval < 0) {
qb_logerr(QB_FAILURE, __FILE__, __LINE__, method,
"Error from ::close File:%s, mode:%d, errno: %d",
_name.c_str(), _mode, errno);
return(QB_FAILURE);
}
}
_name = "";
_fd = 0;
_eof_flag = false;
return(QB_SUCCESS);
}
QBStatus TapFile::write(const QBKey &key, const QBEvent &event)
{
const char *method = "TapFile::write()";
QBStatus status;
string s;
if (!isOpen()) return(QB_STREAMNOTOPEN);
if (_mode == QB_READ) return(QB_WRONGACCESSMODE);
// setup header and trailer fields on the first record written
if (_first_write) {
// setup header
_header.set_message_name(event.GetTypeName());
_header.set_target_decomp_size(_target_block_size);
// write signature and header
_oStream = new google::protobuf::io::CodedOutputStream(_fileOStream);
_oStream->WriteRaw("tapproto",8);
_oStream->WriteRaw("head",4);
_oStream->WriteRaw("none",4);
_oStream->WriteVarint32(_header.ByteSize());
_header.SerializeWithCachedSizes(_oStream);
_pad(_oStream, 0);
delete _oStream;
_oStream = NULL;
// copy the header to the trailer and fill in what we know
_trailer.CopyFrom(_header);
_trailer.set_first_key(key.data().c_str(), key.size());
_trailer.set_message_count(0);
_trailer.set_data_block_count(0);
_trailer.set_uncompressed_bytes(0);
_trailer.set_max_decomp_size(0);
// add descriptors from imported .proto Files to the trailer
for (int i=0; i < event.GetDescriptor()->file()->dependency_count(); i++) {
const FileDescriptor *fd_p = event.GetDescriptor()->file()->dependency(i);
FileDescriptorProto fdproto;
fd_p->CopyTo(&fdproto);
fdproto.SerializeToString(&s);
_header.add_format_descriptor(s.data(), s.size());
}
// add the main File descriptor proto to the trailer
FileDescriptorProto fdproto;
event.GetDescriptor()->file()->CopyTo(&fdproto);
fdproto.SerializeToString(&s);
_header.add_format_descriptor(s.data(), s.size());
// create the ostringstream for the index, write the index block header
// this index can get really large, we'll want to automatically grow
// the data block size as the File gets larger to limit the growth of the index
_indexOstringstream = new ostringstream(ostringstream::binary|ostringstream::app|ostringstream::out);
_indexOstreamOStream = new google::protobuf::io::OstreamOutputStream(_indexOstringstream);
_indexOStream = new google::protobuf::io::CodedOutputStream(_indexOstreamOStream);
_indexOStream->WriteRaw("upix",4);
_indexOStream->WriteRaw("none",4);
// first-write preprocessing is done
_first_write = false;
}
// do we have a current open data block?
if (_dataOStream == NULL) {
// setup the index entry
_index_entry.Clear();
_index_entry.set_first_key(key.data().c_str(), key.size());
_index_entry.set_data_offset(_fileOStream->ByteCount());
_index_entry.set_message_count(0);
// write the data block header
_raw(_fileOStream, "data");
_raw(_fileOStream, "gzip");
// create the GzipOutputStream for the data block
google::protobuf::io::GzipOutputStream::Options options;
options.compression_level = 6;
_dataGzipOStream = new google::protobuf::io::GzipOutputStream(_fileOStream, options);
_dataOStream = new google::protobuf::io::CodedOutputStream(_dataGzipOStream);
}
// serialize the record
_dataOStream->WriteVarint32(key.size());
_dataOStream->WriteRaw(key.data().c_str(),key.size());
_dataOStream->WriteVarint32(event.ByteSize());
event.SerializeWithCachedSizes(_dataOStream);
_index_entry.set_message_count(_index_entry.message_count() + 1);
_trailer.set_last_key(key.data().c_str(), key.size());
// close out this data block if full
if (_dataOStream->ByteCount() >= _target_block_size) {
_flush();
}
return(QB_SUCCESS);
}
void TapFile::_raw(google::protobuf::io::FileOutputStream* fos, const char *string) {
_raw(fos, string, strlen(string));
}
int
TimelineViewWindow::time_to_x(TimeValue const& time) const
{
return int (_raw(time - timeOffset) / timeScale); //////TODO protect against values out-of range
}
