/* ======================== idLobby::SendCompletedSnaps This function will send send off any previously submitted pending snaps if they are ready ======================== */ bool idLobby::SendCompletedSnaps() { assert( lobbyType == GetActingGameStateLobbyType() ); bool sentAllSubmitted = true; for( int p = 0; p < peers.Num(); p++ ) { peer_t& peer = peers[p]; if( !peer.IsConnected() ) { continue; } if( peer.snapProc->PendingSnapReadyToSend() ) { // Check to see if there are any snaps that were submitted that need to be sent out SendCompletedPendingSnap( p ); } else if( IsHost() ) { NET_VERBOSESNAPSHOT_PRINT_LEVEL( 7, va( " ^8Peer %d pendingSnap not ready to send\n", p ) ); } if( !peer.IsConnected() ) // peer may have been dropped in "SendCompletedPendingSnap". ugh. { continue; } if( peer.snapProc->PendingSnapReadyToSend() ) { // If we still have a submitted snap, we know we're not done sentAllSubmitted = false; if( IsHost() ) { NET_VERBOSESNAPSHOT_PRINT_LEVEL( 2, va( " ^2Peer %d did not send all submitted snapshots.\n", p ) ); } } } return sentAllSubmitted; }
/* ======================== idLobby::SubmitPendingSnap ======================== */ bool idLobby::SubmitPendingSnap( int p ) { assert( lobbyType == GetActingGameStateLobbyType() ); peer_t& peer = peers[p]; if( !peer.IsConnected() ) { return false; } // If the peer doesn't have the latest resource list, send it to him before sending any new snapshots if( SendResources( p ) ) { return false; } if( !peer.loaded ) { return false; } if( !peer.snapProc->HasPendingSnap() ) { return false; } int time = Sys_Milliseconds(); int timeFromLastSub = time - peer.lastSnapJobTime; int forceResendTime = session->GetTitleStorageInt( "net_snap_redundant_resend_in_ms", net_snap_redundant_resend_in_ms.GetInteger() ); if( timeFromLastSub < forceResendTime && peer.snapProc->IsBusyConfirmingPartialSnap() ) { return false; } peer.lastSnapJobTime = time; assert( !peer.snapProc->PendingSnapReadyToSend() ); // Submit snapshot delta to jobs peer.snapProc->SubmitPendingSnap( p + 1, objMemory, SNAP_OBJ_JOB_MEMORY, lzwData ); NET_VERBOSESNAPSHOT_PRINT_LEVEL( 2, va( " Submitted snapshot to jobList for peer %d. Since last jobsub: %d\n", p, timeFromLastSub ) ); return true; }
/* ======================== idSnapshotProcessor::ApplySnapshotDelta Apply a snapshot delta to our current basestate, and make that the new base. We can remove all deltas that refer to the basetate we just removed. ======================== */ bool idSnapshotProcessor::ApplySnapshotDelta( int visIndex, int snapshotNumber ) { NET_VERBOSESNAPSHOT_PRINT_LEVEL( 6, va( "idSnapshotProcessor::ApplySnapshotDelta snapshotNumber: %d\n", snapshotNumber ) ); // Sanity check deltas SanityCheckDeltas(); // dump any deltas older than the acknoweledged snapshot, which should only happen if there is packet loss deltas.RemoveOlderThan( snapshotNumber ); if( deltas.Num() == 0 || deltas.ItemSequence( 0 ) != snapshotNumber ) { // this means the snapshot was either already acknowledged or came out of order // On the server, this can happen because the client is continuously/redundantly sending acks // Once the server has ack'd a certain base sequence, it will need to ignore all the redundant ones. // On the client, this will only happen due to out of order, or dropped packets. if( !common->IsServer() ) { // these should be printed every time on the clients // printing on server is not useful / results in tons of spam if( deltas.Num() == 0 ) { NET_VERBOSESNAPSHOT_PRINT( "NET: Got snapshot but ignored... deltas.Num(): %d snapshotNumber: %d \n", deltas.Num(), snapshotNumber ); } else { NET_VERBOSESNAPSHOT_PRINT( "NET: Got snapshot but ignored... deltas.ItemSequence( 0 ): %d != snapshotNumber: %d \n ", deltas.ItemSequence( 0 ), snapshotNumber ); for( int i = 0; i < deltas.Num(); i++ ) { NET_VERBOSESNAPSHOT_PRINT( "%d ", deltas.ItemSequence( i ) ); } NET_VERBOSESNAPSHOT_PRINT( "\n" ); } } return false; } int deltaSequence = 0; int deltaBaseSequence = 0; PeekDeltaSequence( ( const char* )deltas.ItemData( 0 ), deltas.ItemLength( 0 ), deltaSequence, deltaBaseSequence ); assert( deltaSequence == snapshotNumber ); // Make sure compressed sequence number matches that in data queue assert( baseSequence == deltaBaseSequence ); // If this delta isn't based off of our currently ack'd basestate, something is trashed... assert( deltaSequence > baseSequence ); if( baseSequence != deltaBaseSequence ) { // NOTE - This should no longer happen with recent fixes. // We should probably disconnect from the server if this happens. (packets are trashed most likely) NET_VERBOSESNAPSHOT_PRINT( "NET: Got snapshot %d but baseSequence does not match. baseSequence: %d deltaBaseSequence: %d. \n", snapshotNumber, baseSequence, deltaBaseSequence ); return false; } // Apply this delta to our base state if( ApplyDeltaToSnapshot( baseState, ( const char* )deltas.ItemData( 0 ), deltas.ItemLength( 0 ), visIndex ) ) { lastFullSnapBaseSequence = deltaSequence; } baseSequence = deltaSequence; // This is now our new base sequence // Remove deltas that we no longer need RemoveDeltasForOldBaseSequence(); // Sanity check deltas SanityCheckDeltas(); return true; }
/* ======================== idSnapshotProcessor::GetPendingSnapDelta ======================== */ int idSnapshotProcessor::GetPendingSnapDelta( byte* outBuffer, int maxLength ) { assert( PendingSnapReadyToSend() ); if( !verify( jobMemory->lzwInOutData.numlzwDeltas == 1 ) ) { jobMemory->lzwInOutData.numlzwDeltas = 0; return 0; // No more deltas left to send } assert( hasPendingSnap ); jobMemory->lzwInOutData.numlzwDeltas = 0; int size = jobMemory->lzwDeltas[0].size; if( !verify( size != -1 ) ) { #ifdef STRESS_LZW_MEM if( g_maxlwMem < MAX_LZW_MEM ) { g_maxlwMem += 50; g_maxlwMem = Min( g_maxlwMem, MAX_LZW_MEM ); return 0; } #endif // This can happen if there wasn't enough maxlzwMem to process one full obj in a single delta idLib::Error( "GetPendingSnapDelta: Delta failed." ); } uint8* deltaData = &jobMemory->lzwMem[jobMemory->lzwDeltas[0].offset]; int deltaSequence = 0; int deltaBaseSequence = 0; PeekDeltaSequence( ( const char* )deltaData, size, deltaSequence, deltaBaseSequence ); // sanity check: does the compressed data we are about to send have the sequence number we expect assert( deltaSequence == jobMemory->lzwDeltas[0].snapSequence ); if( !verify( size <= maxLength ) ) { idLib::Error( "GetPendingSnapDelta: Size overflow." ); } // Copy to out buffer memcpy( outBuffer, deltaData, size ); // Set the sequence to what this delta actually belongs to assert( jobMemory->lzwDeltas[0].snapSequence == snapSequence + 1 ); snapSequence = jobMemory->lzwDeltas[0].snapSequence; //idLib::Printf( "deltas Num: %i, Size: %i\n", deltas.Num(), deltas.GetDataLength() ); // Copy to delta buffer // NOTE - We don't need to save this delta off if peer has already ack'd this basestate. // This can happen due to the fact that we defer the processing of snap deltas on jobs. // When we start processing a delta, we use the currently ack'd basestate. If while we were processing // the delta, the client acks a new basestate, we can get into this situation. In this case, we simply don't // store the delta, since it will just take up space, and just get removed anyways during ApplySnapshotDelta. // (and cause lots of spam when it sees the delta's basestate doesn't match the current ack'd one) if( deltaBaseSequence >= baseSequence ) { if( !deltas.Append( snapSequence, deltaData, size ) ) { int resendLength = deltas.ItemLength( deltas.Num() - 1 ); if( !verify( resendLength <= maxLength ) ) { idLib::Error( "GetPendingSnapDelta: Size overflow for resend." ); } memcpy( outBuffer, deltas.ItemData( deltas.Num() - 1 ), resendLength ); size = -resendLength; } } if( jobMemory->lzwInOutData.fullSnap ) { // We sent the full snap, we can stop sending this pending snap now... NET_VERBOSESNAPSHOT_PRINT_LEVEL( 5, va( " wrote enough deltas to a full snapshot\n" ) ); // FIXME: peer number? hasPendingSnap = false; partialBaseSequence = -1; } else { partialBaseSequence = deltaBaseSequence; } return size; }
void idLobby::SendSnapshotToPeer( idSnapShot& ss, int p ) { assert( lobbyType == GetActingGameStateLobbyType() ); peer_t& peer = peers[p]; if( net_forceDropSnap.GetBool() ) { net_forceDropSnap.SetBool( false ); return; } if( peer.pauseSnapshots ) { return; } int time = Sys_Milliseconds(); const int throttleMode = session->GetTitleStorageInt( "net_peer_throttle_mode", net_peer_throttle_mode.GetInteger() ); // Real peer throttling based on performance // -We throttle before sending to jobs rather than before sending if( ( throttleMode == 1 || throttleMode == 3 ) && peer.throttledSnapRate > 0 ) { if( time - peer.lastSnapJobTime < peer.throttledSnapRate / 1000 ) // fixme /1000 { // This peer is throttled, skip his snap shot NET_VERBOSESNAPSHOT_PRINT_LEVEL( 2, va( "NET: Throttling peer %d.Skipping snapshot. Time elapsed: %d peer snap rate: %d\n", p, ( time - peer.lastSnapJobTime ), peer.throttledSnapRate ) ); return; } } if( throttleMode != 0 ) { DetectSaturation( p ); } if( peer.maxSnapBps >= 0.0f && ( throttleMode == 2 || throttleMode == 3 ) ) { if( peer.packetProc->GetOutgoingRateBytes() > peer.maxSnapBps ) { return; } } // TrySetPendingSnapshot will try to set the new pending snap. // TrySetPendingSnapshot won't do anything until the last snap set was fully sent out. if( peer.snapProc->TrySetPendingSnapshot( ss ) ) { NET_VERBOSESNAPSHOT_PRINT_LEVEL( 2, va( " ^8Set next pending snapshot peer %d\n", 0 ) ); peer.numSnapsSent++; idSnapShot* baseState = peers[p].snapProc->GetBaseState(); if( verify( baseState != NULL ) ) { baseState->UpdateExpectedSeq( peers[p].snapProc->GetSnapSequence() ); } } else { NET_VERBOSESNAPSHOT_PRINT_LEVEL( 2, va( " ^2FAILED Set next pending snapshot peer %d\n", 0 ) ); } // We send out the pending snap, which could be the most recent, or an old one that hasn't fully been sent // We don't send immediately, since we have to coordinate sending snaps for all peers in same place considering jobs. peer.needToSubmitPendingSnap = true; }
/* ======================== idLobby::SendCompletedPendingSnap ======================== */ void idLobby::SendCompletedPendingSnap( int p ) { assert( lobbyType == GetActingGameStateLobbyType() ); int time = Sys_Milliseconds(); peer_t& peer = peers[p]; if( !peer.IsConnected() ) { return; } if( peer.snapProc == NULL || !peer.snapProc->PendingSnapReadyToSend() ) { return; } // If we have a pending snap ready to send, we better have a pending snap assert( peer.snapProc->HasPendingSnap() ); // Get the snap data blob now, even if we don't send it. // This is somewhat wasteful, but we have to do this to keep the snap job pipe ready to keep doing work // If we don't do this, this peer will cause other peers to be starved of snapshots, when they may very well be ready to send a snap byte buffer[ MAX_SNAP_SIZE ]; int maxLength = sizeof( buffer ) - peer.packetProc->GetReliableDataSize() - 128; int size = peer.snapProc->GetPendingSnapDelta( buffer, maxLength ); if( !CanSendMoreData( p ) ) { return; } // Can't send anymore snapshots until all fragments are sent if( peer.packetProc->HasMoreFragments() ) { return; } // If the peer doesn't have the latest resource list, send it to him before sending any new snapshots if( SendResources( p ) ) { return; } int timeFromJobSub = time - peer.lastSnapJobTime; int timeFromLastSend = time - peer.lastSnapTime; if( timeFromLastSend > 0 ) { peer.snapHz = 1000.0f / ( float )timeFromLastSend; } else { peer.snapHz = 0.0f; } if( net_snapshot_send_warntime.GetInteger() > 0 && peer.lastSnapTime != 0 && net_snapshot_send_warntime.GetInteger() < timeFromLastSend ) { idLib::Printf( "NET: Took %d ms to send peer %d snapshot\n", timeFromLastSend, p ); } if( peer.throttleSnapsForXSeconds != 0 ) { if( time < peer.throttleSnapsForXSeconds ) { return; } // If we were trying to recover ping, see if we succeeded if( peer.recoverPing != 0 ) { if( peer.lastPingRtt >= peer.recoverPing ) { peer.failedPingRecoveries++; } else { const int peer_throttle_minSnapSeq = session->GetTitleStorageInt( "net_peer_throttle_minSnapSeq", net_peer_throttle_minSnapSeq.GetInteger() ); if( peer.snapProc->GetFullSnapBaseSequence() > idSnapshotProcessor::INITIAL_SNAP_SEQUENCE + peer_throttle_minSnapSeq ) { // If throttling recovered the ping int maxRate = common->GetSnapRate() * session->GetTitleStorageInt( "net_peer_throttle_maxSnapRate", net_peer_throttle_maxSnapRate.GetInteger() ); peer.throttledSnapRate = idMath::ClampInt( common->GetSnapRate(), maxRate, peer.throttledSnapRate + common->GetSnapRate() ); } } } peer.throttleSnapsForXSeconds = 0; } peer.lastSnapTime = time; if( size != 0 ) { if( size > 0 ) { NET_VERBOSESNAPSHOT_PRINT_LEVEL( 3, va( "NET: (peer %d) Sending snapshot %d delta'd against %d. Since JobSub: %d Since LastSend: %d. Size: %d\n", p, peer.snapProc->GetSnapSequence(), peer.snapProc->GetBaseSequence(), timeFromJobSub, timeFromLastSend, size ) ); ProcessOutgoingMsg( p, buffer, size, false, 0 ); } else if( size < 0 ) // Size < 0 indicates the delta buffer filled up { // There used to be code here that would disconnect peers if they were in game and filled up the buffer // This was causing issues in the playtests we were running (Doom 4 MP) and after some conversation // determined that it was not needed since a timeout mechanism has been added since ProcessOutgoingMsg( p, buffer, -size, false, 0 ); if( peer.snapProc != NULL ) { NET_VERBOSESNAPSHOT_PRINT( "NET: (peerNum: %d - name: %s) Resending last snapshot delta %d because his delta list filled up. Since JobSub: %d Since LastSend: %d Delta Size: %d\n", p, GetPeerName( p ), peer.snapProc->GetSnapSequence(), timeFromJobSub, timeFromLastSend, size ); } } } // We calculate what our outgoing rate was for each sequence, so we can have a relative comparison // for when the client reports what his downstream was in the same timeframe if( IsHost() && peer.snapProc != NULL && peer.snapProc->GetSnapSequence() > 0 ) { //NET_VERBOSE_PRINT("^8 %i Rate: %.2f SnapSeq: %d GetBaseSequence: %d\n", lastAppendedSequence, peer.packetProc->GetOutgoingRateBytes(), peer.snapProc->GetSnapSequence(), peer.snapProc->GetBaseSequence() ); peer.sentBpsHistory[ peer.snapProc->GetSnapSequence() % MAX_BPS_HISTORY ] = peer.packetProc->GetOutgoingRateBytes(); } }