void ProcessEncodedFrame(List<DataPacket> &packets, List<PacketType> &packetTypes, DWORD outputTimestamp, int &ctsOffset, mfxU32 wait=0) { if(!encoded_tasks.Num()) return; encode_task& task = encode_tasks[encoded_tasks[0]]; auto& sp = task.sp; if(MFXVideoCORE_SyncOperation(session, sp, wait) != MFX_ERR_NONE) return; mfxBitstream& bs = task.bs; List<x264_nal_t> nalOut; mfxU8 *start = bs.Data + bs.DataOffset, *end = bs.Data + bs.DataOffset + bs.DataLength; static mfxU8 start_seq[] = {0, 0, 1}; start = std::search(start, end, start_seq, start_seq+3); while(start != end) { decltype(start) next = std::search(start+1, end, start_seq, start_seq+3); x264_nal_t nal; nal.i_ref_idc = start[3]>>5; nal.i_type = start[3]&0x1f; if(nal.i_type == NAL_SLICE_IDR) nal.i_ref_idc = NAL_PRIORITY_HIGHEST; nal.p_payload = start; nal.i_payload = int(next-start); nalOut << nal; start = next; } size_t nalNum = nalOut.Num(); packets.Clear(); ClearPackets(); INT64 dts; if(bUsingDecodeTimestamp && bs.DecodeTimeStamp != MFX_TIMESTAMP_UNKNOWN) { dts = msFromTimestamp(bs.DecodeTimeStamp); } else dts = outputTimestamp; INT64 in_pts = msFromTimestamp(task.surf.Data.TimeStamp), out_pts = msFromTimestamp(bs.TimeStamp); if(!bFirstFrameProcessed && nalNum) { delayOffset = -dts; bFirstFrameProcessed = true; } INT64 ts = INT64(outputTimestamp); int timeOffset; if(bDupeFrames) { //if frame duplication is being used, the shift will be insignificant, so just don't bother adjusting audio timeOffset = int(out_pts-dts); timeOffset += frameShift; if(nalNum && timeOffset < 0) { frameShift -= timeOffset; timeOffset = 0; } } else { timeOffset = int(out_pts+delayOffset-ts); timeOffset += ctsOffset; //dynamically adjust the CTS for the stream if it gets lower than the current value //(thanks to cyrus for suggesting to do this instead of a single shift) if(nalNum && timeOffset < 0) { ctsOffset -= timeOffset; timeOffset = 0; } } //Log(TEXT("inpts: %005d, dts: %005d, pts: %005d, timestamp: %005d, offset: %005d, newoffset: %005d"), task.surf.Data.TimeStamp/90, dts, bs.TimeStamp/90, outputTimestamp, timeOffset, bs.TimeStamp/90-dts); timeOffset = htonl(timeOffset); BYTE *timeOffsetAddr = ((BYTE*)&timeOffset)+1; VideoPacket *newPacket = NULL; PacketType bestType = PacketType_VideoDisposable; bool bFoundFrame = false; for(int i=0; i<nalNum; i++) { x264_nal_t &nal = nalOut[i]; if(nal.i_type == NAL_SEI) { BYTE *skip = nal.p_payload; while(*(skip++) != 0x1); int skipBytes = (int)(skip-nal.p_payload); int newPayloadSize = (nal.i_payload-skipBytes); if (nal.p_payload[skipBytes+1] == 0x5) { SEIData.Clear(); BufferOutputSerializer packetOut(SEIData); packetOut.OutputDword(htonl(newPayloadSize)); packetOut.Serialize(nal.p_payload+skipBytes, newPayloadSize); } else { if (!newPacket) newPacket = CurrentPackets.CreateNew(); BufferOutputSerializer packetOut(newPacket->Packet); packetOut.OutputDword(htonl(newPayloadSize)); packetOut.Serialize(nal.p_payload+skipBytes, newPayloadSize); } } /*else if(nal.i_type == NAL_FILLER) //QSV does not produce NAL_FILLER { BYTE *skip = nal.p_payload; while(*(skip++) != 0x1); int skipBytes = (int)(skip-nal.p_payload); int newPayloadSize = (nal.i_payload-skipBytes); if (!newPacket) newPacket = CurrentPackets.CreateNew(); BufferOutputSerializer packetOut(newPacket->Packet); packetOut.OutputDword(htonl(newPayloadSize)); packetOut.Serialize(nal.p_payload+skipBytes, newPayloadSize); }*/ else if(nal.i_type == NAL_SLICE_IDR || nal.i_type == NAL_SLICE) { BYTE *skip = nal.p_payload; while(*(skip++) != 0x1); int skipBytes = (int)(skip-nal.p_payload); if (!newPacket) newPacket = CurrentPackets.CreateNew(); if (!bFoundFrame) { newPacket->Packet.Insert(0, (nal.i_type == NAL_SLICE_IDR) ? 0x17 : 0x27); newPacket->Packet.Insert(1, 1); newPacket->Packet.InsertArray(2, timeOffsetAddr, 3); bFoundFrame = true; } int newPayloadSize = (nal.i_payload-skipBytes); BufferOutputSerializer packetOut(newPacket->Packet); packetOut.OutputDword(htonl(newPayloadSize)); packetOut.Serialize(nal.p_payload+skipBytes, newPayloadSize); switch(nal.i_ref_idc) { case NAL_PRIORITY_DISPOSABLE: bestType = MAX(bestType, PacketType_VideoDisposable); break; case NAL_PRIORITY_LOW: bestType = MAX(bestType, PacketType_VideoLow); break; case NAL_PRIORITY_HIGH: bestType = MAX(bestType, PacketType_VideoHigh); break; case NAL_PRIORITY_HIGHEST: bestType = MAX(bestType, PacketType_VideoHighest); break; } } /*else if(nal.i_type == NAL_SPS) { VideoPacket *newPacket = CurrentPackets.CreateNew(); BufferOutputSerializer headerOut(newPacket->Packet); headerOut.OutputByte(0x17); headerOut.OutputByte(0); headerOut.Serialize(timeOffsetAddr, 3); headerOut.OutputByte(1); headerOut.Serialize(nal.p_payload+5, 3); headerOut.OutputByte(0xff); headerOut.OutputByte(0xe1); headerOut.OutputWord(htons(nal.i_payload-4)); headerOut.Serialize(nal.p_payload+4, nal.i_payload-4); x264_nal_t &pps = nalOut[i+1]; //the PPS always comes after the SPS headerOut.OutputByte(1); headerOut.OutputWord(htons(pps.i_payload-4)); headerOut.Serialize(pps.p_payload+4, pps.i_payload-4); }*/ else continue; } packetTypes << bestType; packets.SetSize(CurrentPackets.Num()); for(UINT i=0; i<packets.Num(); i++) { packets[i].lpPacket = CurrentPackets[i].Packet.Array(); packets[i].size = CurrentPackets[i].Packet.Num(); } msdk_locked_tasks << encoded_tasks[0]; encoded_tasks.Remove(0); }
void ProcessEncodedFrame(List<DataPacket> &packets, List<PacketType> &packetTypes, DWORD outputTimestamp, mfxU32 wait=0) { if(!filled_bitstream_waiter.wait_for(2, wait)) return; uint32_t index = 0; { auto lock = lock_mutex(filled_bitstream); index = *filled_bitstream; *filled_bitstream = -1; } encode_task& task = encode_tasks[index]; mfxBitstream& bs = task.bs; List<x264_nal_t> nalOut; mfxU8 *start, *end; { bitstream_info &info = bs_info[index]; bs.TimeStamp = info.time_stamp; bs.DataLength = info.data_length; bs.DataOffset = info.data_offset; bs.PicStruct = info.pic_struct; bs.FrameType = info.frame_type; } start = bs.Data + bs.DataOffset; end = bs.Data + bs.DataOffset + bs.DataLength; const static mfxU8 start_seq[] = {0, 0, 1}; start = std::search(start, end, start_seq, start_seq+3); while(start != end) { decltype(start) next = std::search(start+1, end, start_seq, start_seq+3); x264_nal_t nal; nal.i_ref_idc = start[3]>>5; nal.i_type = start[3]&0x1f; if(nal.i_type == NAL_SLICE_IDR) nal.i_ref_idc = NAL_PRIORITY_HIGHEST; else if(nal.i_type == NAL_SLICE) { switch(bs.FrameType & (MFX_FRAMETYPE_REF | (MFX_FRAMETYPE_S-1))) { case MFX_FRAMETYPE_REF|MFX_FRAMETYPE_I: case MFX_FRAMETYPE_REF|MFX_FRAMETYPE_P: nal.i_ref_idc = NAL_PRIORITY_HIGH; break; case MFX_FRAMETYPE_REF|MFX_FRAMETYPE_B: nal.i_ref_idc = NAL_PRIORITY_LOW; break; case MFX_FRAMETYPE_B: nal.i_ref_idc = NAL_PRIORITY_DISPOSABLE; break; default: Log(TEXT("Unhandled frametype %u"), bs.FrameType); } } start[3] = ((nal.i_ref_idc<<5)&0x60) | nal.i_type; nal.p_payload = start; nal.i_payload = int(next-start); nalOut << nal; start = next; } size_t nalNum = nalOut.Num(); packets.Clear(); ClearPackets(); INT64 dts = outputTimestamp; INT64 in_pts = msFromTimestamp(task.surf.Data.TimeStamp), out_pts = msFromTimestamp(bs.TimeStamp); if(!bFirstFrameProcessed && nalNum) { delayOffset = -dts; bFirstFrameProcessed = true; } INT64 ts = INT64(outputTimestamp); int timeOffset; //if frame duplication is being used, the shift will be insignificant, so just don't bother adjusting audio timeOffset = int(out_pts-dts); timeOffset += frameShift; if(nalNum && timeOffset < 0) { frameShift -= timeOffset; timeOffset = 0; } //Log(TEXT("inpts: %005d, dts: %005d, pts: %005d, timestamp: %005d, offset: %005d, newoffset: %005d"), task.surf.Data.TimeStamp/90, dts, bs.TimeStamp/90, outputTimestamp, timeOffset, bs.TimeStamp/90-dts); timeOffset = htonl(timeOffset); BYTE *timeOffsetAddr = ((BYTE*)&timeOffset)+1; VideoPacket *newPacket = NULL; PacketType bestType = PacketType_VideoDisposable; bool bFoundFrame = false; for(unsigned i=0; i<nalNum; i++) { x264_nal_t &nal = nalOut[i]; if(nal.i_type == NAL_SEI) { BYTE *skip = nal.p_payload; while(*(skip++) != 0x1); int skipBytes = (int)(skip-nal.p_payload); int newPayloadSize = (nal.i_payload-skipBytes); BYTE *sei_start = skip+1; while(sei_start < (nal.p_payload+nal.i_payload)) { BYTE *sei = sei_start; int sei_type = 0; while(*sei == 0xff) { sei_type += 0xff; sei += 1; } sei_type += *sei++; int payload_size = 0; while(*sei == 0xff) { payload_size += 0xff; sei += 1; } payload_size += *sei++; const static BYTE emulation_prevention_pattern[] = {0, 0, 3}; BYTE *search = sei; for(BYTE *search = sei;;) { search = std::search(search, sei+payload_size, emulation_prevention_pattern, emulation_prevention_pattern+3); if(search == sei+payload_size) break; payload_size += 1; search += 3; } int sei_size = (int)(sei-sei_start) + payload_size; sei_start[-1] = NAL_SEI; if(sei_type == SEI_USER_DATA_UNREGISTERED) { SEIData.Clear(); BufferOutputSerializer packetOut(SEIData); packetOut.OutputDword(htonl(sei_size+1)); packetOut.Serialize(sei_start-1, sei_size+1); } else { if (!newPacket) newPacket = CurrentPackets.CreateNew(); BufferOutputSerializer packetOut(newPacket->Packet); packetOut.OutputDword(htonl(sei_size+1)); packetOut.Serialize(sei_start-1, sei_size+1); } sei_start += sei_size; } } else if(nal.i_type == NAL_AUD) { BYTE *skip = nal.p_payload; while(*(skip++) != 0x1); int skipBytes = (int)(skip-nal.p_payload); int newPayloadSize = (nal.i_payload-skipBytes); if (!newPacket) newPacket = CurrentPackets.CreateNew(); BufferOutputSerializer packetOut(newPacket->Packet); packetOut.OutputDword(htonl(newPayloadSize)); packetOut.Serialize(nal.p_payload+skipBytes, newPayloadSize); } else if(nal.i_type == NAL_SLICE_IDR || nal.i_type == NAL_SLICE) { BYTE *skip = nal.p_payload; while(*(skip++) != 0x1); int skipBytes = (int)(skip-nal.p_payload); if (!newPacket) newPacket = CurrentPackets.CreateNew(); if (!bFoundFrame) { newPacket->Packet.Insert(0, (nal.i_type == NAL_SLICE_IDR) ? 0x17 : 0x27); newPacket->Packet.Insert(1, 1); newPacket->Packet.InsertArray(2, timeOffsetAddr, 3); bFoundFrame = true; } int newPayloadSize = (nal.i_payload-skipBytes); BufferOutputSerializer packetOut(newPacket->Packet); packetOut.OutputDword(htonl(newPayloadSize)); packetOut.Serialize(nal.p_payload+skipBytes, newPayloadSize); switch(nal.i_ref_idc) { case NAL_PRIORITY_DISPOSABLE: bestType = MAX(bestType, PacketType_VideoDisposable); break; case NAL_PRIORITY_LOW: bestType = MAX(bestType, PacketType_VideoLow); break; case NAL_PRIORITY_HIGH: bestType = MAX(bestType, PacketType_VideoHigh); break; case NAL_PRIORITY_HIGHEST: bestType = MAX(bestType, PacketType_VideoHighest); break; } } /*else if(nal.i_type == NAL_SPS) { VideoPacket *newPacket = CurrentPackets.CreateNew(); BufferOutputSerializer headerOut(newPacket->Packet); headerOut.OutputByte(0x17); headerOut.OutputByte(0); headerOut.Serialize(timeOffsetAddr, 3); headerOut.OutputByte(1); headerOut.Serialize(nal.p_payload+5, 3); headerOut.OutputByte(0xff); headerOut.OutputByte(0xe1); headerOut.OutputWord(htons(nal.i_payload-4)); headerOut.Serialize(nal.p_payload+4, nal.i_payload-4); x264_nal_t &pps = nalOut[i+1]; //the PPS always comes after the SPS headerOut.OutputByte(1); headerOut.OutputWord(htons(pps.i_payload-4)); headerOut.Serialize(pps.p_payload+4, pps.i_payload-4); }*/ else continue; } packetTypes << bestType; packets.SetSize(CurrentPackets.Num()); for(UINT i=0; i<packets.Num(); i++) { packets[i].lpPacket = CurrentPackets[i].Packet.Array(); packets[i].size = CurrentPackets[i].Packet.Num(); } idle_tasks << index; assert(queued_tasks[0] == index); queued_tasks.Remove(0); }