Skip to content

Commit

Permalink
Fix glitches that sometimes happen near the scene changes. (#3707)
Browse files Browse the repository at this point in the history
* Add missing iLastWrittenPOC setting in unbuffered reordering.

In CWelsDecoder::ReorderPicturesInDisplay(), iLastWrittenPOC was
not set in unbuffered-reordering case. Due to this problem, it
sometimes reordered the frames incorrectly. This patch fixes the
issue.

* Prevent frame buffer from overwrite at GOP change.

When bNewSeqBegin got true, iRefCount was forcibly reset even if the
buffer was still used for reordering. Due to this problem, the buffer
in use was sometimes overwritten with newly decoded frame. This commit
is for fixing that problem.

* Fix test failure due to the issue of the previous commit.

* Fix glitch occurs when multiple scene changes happen in a short time.

Previously, when more than 2 frame sequences exist in the reordering
buffer, the reordering does not work correctly. This is because the
management based on bLastGOP flag does not work in this case.
Instead, this patch introduces frame sequence number (iSeqNum) to
distingish frames belonging to each sequence and drops bLastGOP flag.
  • Loading branch information
tyan0 authored Jan 19, 2024
1 parent cfbd589 commit 4f01c15
Show file tree
Hide file tree
Showing 7 changed files with 38 additions and 92 deletions.
6 changes: 4 additions & 2 deletions codec/decoder/core/inc/decoder_context.h
Original file line number Diff line number Diff line change
Expand Up @@ -285,14 +285,15 @@ typedef struct tagPictInfo {
int32_t iPOC;
int32_t iPicBuffIdx;
uint32_t uiDecodingTimeStamp;
bool bLastGOP;
int32_t iSeqNum;
} SPictInfo, *PPictInfo;

typedef struct tagPictReoderingStatus {
int32_t iPictInfoIndex;
int32_t iMinSeqNum;
int32_t iMinPOC;
int32_t iNumOfPicts;
int32_t iLastGOPRemainPicts;
int32_t iLastWrittenSeqNum;
int32_t iLastWrittenPOC;
int32_t iLargestBufferedPicIndex;
bool bHasBSlice;
Expand Down Expand Up @@ -431,6 +432,7 @@ typedef struct TagWelsDecoderContext {
#endif
bool bNewSeqBegin;
bool bNextNewSeqBegin;
int32_t iSeqNum;

//for Parse only
bool bFramePending;
Expand Down
1 change: 1 addition & 0 deletions codec/decoder/core/inc/picture.h
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ struct SPicture {
bool bUsedAsRef; //for ref pic management
bool bIsLongRef; // long term reference frame flag //for ref pic management
int8_t iRefCount;
void (*pSetUnRef)(WelsDec::SPicture*);

bool bIsComplete; // indicate whether current picture is complete, not from EC
/*******************************for future use****************************/
Expand Down
5 changes: 3 additions & 2 deletions codec/decoder/core/src/decoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ static int32_t IncreasePicBuff (PWelsDecoderContext pCtx, PPicBuff* ppPicBuf, co
pPicNewBuf->ppPic[i]->bUsedAsRef = false;
pPicNewBuf->ppPic[i]->bIsLongRef = false;
pPicNewBuf->ppPic[i]->iRefCount = 0;
pPicNewBuf->ppPic[i]->pSetUnRef = NULL;
pPicNewBuf->ppPic[i]->bIsComplete = false;
}
// remove old PicBuf
Expand Down Expand Up @@ -240,6 +241,7 @@ static int32_t DecreasePicBuff (PWelsDecoderContext pCtx, PPicBuff* ppPicBuf, co
pPicNewBuf->ppPic[i]->bUsedAsRef = false;
pPicNewBuf->ppPic[i]->bIsLongRef = false;
pPicNewBuf->ppPic[i]->iRefCount = 0;
pPicNewBuf->ppPic[i]->pSetUnRef = NULL;
pPicNewBuf->ppPic[i]->bIsComplete = false;
}
// remove old PicBuf
Expand Down Expand Up @@ -296,11 +298,9 @@ void ResetReorderingPictureBuffers (PPictReoderingStatus pPictReoderingStatus, P
pPictReoderingStatus->iPictInfoIndex = 0;
pPictReoderingStatus->iMinPOC = IMinInt32;
pPictReoderingStatus->iNumOfPicts = 0;
pPictReoderingStatus->iLastGOPRemainPicts = 0;
pPictReoderingStatus->iLastWrittenPOC = IMinInt32;
pPictReoderingStatus->iLargestBufferedPicIndex = 0;
for (int32_t i = 0; i < pictInfoListCount; ++i) {
pPictInfo[i].bLastGOP = false;
pPictInfo[i].iPOC = IMinInt32;
}
pPictInfo->sBufferInfo.iBufferStatus = 0;
Expand Down Expand Up @@ -621,6 +621,7 @@ int32_t WelsOpenDecoder (PWelsDecoderContext pCtx, SLogContext* pLogCtx) {
pCtx->bPrintFrameErrorTraceFlag = true;
pCtx->iIgnoredErrorInfoPacketCount = 0;
pCtx->bFrameFinish = true;
pCtx->iSeqNum = 0;
return iRet;
}

Expand Down
2 changes: 2 additions & 0 deletions codec/decoder/core/src/decoder_core.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2261,6 +2261,8 @@ int32_t WelsDecodeInitAccessUnitStart (PWelsDecoderContext pCtx, SBufferInfo* pD
pCtx->bAuReadyFlag = false;
pCtx->pLastDecPicInfo->bLastHasMmco5 = false;
bool bTmpNewSeqBegin = CheckNewSeqBeginAndUpdateActiveLayerSps (pCtx);
if (bTmpNewSeqBegin)
pCtx->iSeqNum++;
pCtx->bNewSeqBegin = pCtx->bNewSeqBegin || bTmpNewSeqBegin;
iErr = WelsDecodeAccessUnitStart (pCtx);
GetVclNalTemporalId (pCtx);
Expand Down
12 changes: 7 additions & 5 deletions codec/decoder/core/src/manage_dec_ref.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,9 @@ int32_t GetLTRFrameIndex (PRefPic pRefPic, int32_t iAncLTRFrameNum);
static int32_t RemainOneBufferInDpbForEC (PWelsDecoderContext pCtx, PRefPic pRefPic);

static void SetUnRef (PPicture pRef) {
if (NULL != pRef) {
if (pRef == NULL) return;

if (pRef->iRefCount <= 0) {
pRef->bUsedAsRef = false;
pRef->bIsLongRef = false;
pRef->iFrameNum = -1;
Expand All @@ -81,19 +83,19 @@ static void SetUnRef (PPicture pRef) {
pRef->iSpsId = -1;
pRef->bIsComplete = false;
pRef->iRefCount = 0;
pRef->pSetUnRef = NULL;

if (pRef->eSliceType == I_SLICE) {
return;
}
int32_t lists = pRef->eSliceType == P_SLICE ? 1 : 2;
for (int32_t i = 0; i < MAX_DPB_COUNT; ++i) {
for (int32_t list = 0; list < lists; ++list) {
if (pRef->pRefPic[list][i] != NULL) {
pRef->pRefPic[list][i]->iRefCount = 0;
pRef->pRefPic[list][i] = NULL;
}
pRef->pRefPic[list][i] = NULL;
}
}
} else {
pRef->pSetUnRef = SetUnRef;
}
}

Expand Down
1 change: 1 addition & 0 deletions codec/decoder/core/src/pic_queue.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ PPicture AllocPicture (PWelsDecoderContext pCtx, const int32_t kiPicWidth, const
pPic->iHeightInPixel = kiPicHeight;
pPic->iFrameNum = -1;
pPic->iRefCount = 0;
pPic->pSetUnRef = NULL;

uint32_t uiMbWidth = (kiPicWidth + 15) >> 4;
uint32_t uiMbHeight = (kiPicHeight + 15) >> 4;
Expand Down
103 changes: 20 additions & 83 deletions codec/decoder/plus/src/welsDecoderExt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -993,46 +993,17 @@ void CWelsDecoder::BufferingReadyPicture (PWelsDecoderContext pCtx, unsigned cha
if (pCtx->pSliceHeader->eSliceType == B_SLICE) {
m_sReoderingStatus.bHasBSlice = true;
}
if (m_sReoderingStatus.iNumOfPicts && pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb
&& pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb->bNewSeqBegin) {
m_sReoderingStatus.iLastGOPRemainPicts = m_sReoderingStatus.iNumOfPicts;

for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (m_sPictInfoList[i].iPOC > IMinInt32) {
m_sPictInfoList[i].bLastGOP = true;
}
}
} else {
if (m_sReoderingStatus.iNumOfPicts > 0) {
//This can happen when decoder moves to next GOP without being able to decoder first picture PicOrderCntLsb = 0
bool hasGOPChanged = false;
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (m_sPictInfoList[i].iPOC == pCtx->pSliceHeader->iPicOrderCntLsb) {
hasGOPChanged = true;
break;
}
}
if (hasGOPChanged) {
m_sReoderingStatus.iLastGOPRemainPicts = m_sReoderingStatus.iNumOfPicts;
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (m_sPictInfoList[i].iPOC > IMinInt32) {
m_sPictInfoList[i].bLastGOP = true;
}
}
}
}
}
}
for (int32_t i = 0; i < 16; ++i) {
if (m_sPictInfoList[i].iPOC == IMinInt32) {
memcpy (&m_sPictInfoList[i].sBufferInfo, pDstInfo, sizeof (SBufferInfo));
m_sPictInfoList[i].iPOC = pCtx->pSliceHeader->iPicOrderCntLsb;
m_sPictInfoList[i].iSeqNum = pCtx->iSeqNum;
m_sPictInfoList[i].uiDecodingTimeStamp = pCtx->uiDecodingTimeStamp;
if (pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb != NULL) {
m_sPictInfoList[i].iPicBuffIdx = pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb->iPicBuffIdx;
if (GetThreadCount (pCtx) <= 1) ++pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb->iRefCount;
}
m_sPictInfoList[i].bLastGOP = false;
m_iLastBufferedIdx = i;
pDstInfo->iBufferStatus = 0;
++m_sReoderingStatus.iNumOfPicts;
Expand All @@ -1050,65 +1021,24 @@ void CWelsDecoder::ReleaseBufferedReadyPictureReorder (PWelsDecoderContext pCtx,
if (pCtx == NULL && m_iThreadCount <= 1) {
pCtx = m_pDecThrCtx[0].pCtx;
}
if (m_sReoderingStatus.iLastGOPRemainPicts > 0) {
m_sReoderingStatus.iMinPOC = IMinInt32;
int32_t firstValidIdx = -1;
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (m_sReoderingStatus.iMinPOC == IMinInt32 && m_sPictInfoList[i].iPOC > IMinInt32 && m_sPictInfoList[i].bLastGOP) {
m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
m_sReoderingStatus.iPictInfoIndex = i;
firstValidIdx = i;
break;
}
}
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (i == firstValidIdx) continue;
if (m_sPictInfoList[i].iPOC > IMinInt32 && m_sPictInfoList[i].iPOC < m_sReoderingStatus.iMinPOC
&& m_sPictInfoList[i].bLastGOP) {
m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
m_sReoderingStatus.iPictInfoIndex = i;
}
}
m_sReoderingStatus.iLastWrittenPOC = m_sReoderingStatus.iMinPOC;
#if defined (_DEBUG)
#ifdef _MOTION_VECTOR_DUMP_
fprintf (stderr, "Output POC: #%d uiDecodingTimeStamp=%d\n", m_sReoderingStatus.iLastWrittenPOC,
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].uiDecodingTimeStamp);
#endif
#endif
memcpy (pDstInfo, &m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].sBufferInfo, sizeof (SBufferInfo));
ppDst[0] = pDstInfo->pDst[0];
ppDst[1] = pDstInfo->pDst[1];
ppDst[2] = pDstInfo->pDst[2];
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC = IMinInt32;
if (pPicBuff != NULL) {
PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
--pPic->iRefCount;
}
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
m_sReoderingStatus.iMinPOC = IMinInt32;
--m_sReoderingStatus.iNumOfPicts;
--m_sReoderingStatus.iLastGOPRemainPicts;
if (m_sReoderingStatus.iLastGOPRemainPicts == 0) {
m_sReoderingStatus.iLastWrittenPOC = IMinInt32;
}
return;
}
if (m_sReoderingStatus.iNumOfPicts > 0) {
m_sReoderingStatus.iMinPOC = IMinInt32;
int32_t firstValidIdx = -1;
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (m_sReoderingStatus.iMinPOC == IMinInt32 && m_sPictInfoList[i].iPOC > IMinInt32) {
m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
m_sReoderingStatus.iMinSeqNum = m_sPictInfoList[i].iSeqNum;
m_sReoderingStatus.iPictInfoIndex = i;
firstValidIdx = i;
break;
}
}
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (i == firstValidIdx) continue;
if (m_sPictInfoList[i].iPOC > IMinInt32 && m_sPictInfoList[i].iPOC < m_sReoderingStatus.iMinPOC) {
if (m_sPictInfoList[i].iPOC > IMinInt32
&& ((m_sPictInfoList[i].iSeqNum == m_sReoderingStatus.iMinSeqNum) ? (m_sPictInfoList[i].iPOC < m_sReoderingStatus.iMinPOC) : (m_sPictInfoList[i].iSeqNum - m_sReoderingStatus.iMinSeqNum < 0))) {
m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
m_sReoderingStatus.iMinSeqNum = m_sPictInfoList[i].iSeqNum;
m_sReoderingStatus.iPictInfoIndex = i;
}
}
Expand All @@ -1117,12 +1047,15 @@ void CWelsDecoder::ReleaseBufferedReadyPictureReorder (PWelsDecoderContext pCtx,
bool isReady = true;
if (!isFlush) {
int32_t iLastPOC = pCtx != NULL ? pCtx->pSliceHeader->iPicOrderCntLsb : m_sPictInfoList[m_iLastBufferedIdx].iPOC;
int32_t iLastSeqNum = pCtx != NULL ? pCtx->iSeqNum : m_sPictInfoList[m_iLastBufferedIdx].iSeqNum;
isReady = (m_sReoderingStatus.iLastWrittenPOC > IMinInt32
&& m_sReoderingStatus.iMinPOC - m_sReoderingStatus.iLastWrittenPOC <= 1)
|| m_sReoderingStatus.iMinPOC < iLastPOC;
|| m_sReoderingStatus.iMinPOC < iLastPOC
|| m_sReoderingStatus.iMinSeqNum - iLastSeqNum < 0;
}
if (isReady) {
m_sReoderingStatus.iLastWrittenPOC = m_sReoderingStatus.iMinPOC;
m_sReoderingStatus.iLastWrittenSeqNum = m_sReoderingStatus.iMinSeqNum;
#if defined (_DEBUG)
#ifdef _MOTION_VECTOR_DUMP_
fprintf (stderr, "Output POC: #%d uiDecodingTimeStamp=%d\n", m_sReoderingStatus.iLastWrittenPOC,
Expand All @@ -1140,9 +1073,10 @@ void CWelsDecoder::ReleaseBufferedReadyPictureReorder (PWelsDecoderContext pCtx,
{
PPicture pPic = pPicBuff->ppPic[iPicBuffIdx];
--pPic->iRefCount;
if (pPic->iRefCount <= 0 && pPic->pSetUnRef)
pPic->pSetUnRef(pPic);
}
}
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
m_sReoderingStatus.iMinPOC = IMinInt32;
--m_sReoderingStatus.iNumOfPicts;
}
Expand Down Expand Up @@ -1178,6 +1112,7 @@ void CWelsDecoder::ReleaseBufferedReadyPictureNoReorder(PWelsDecoderContext pCtx
#endif
#endif
m_sReoderingStatus.iLastWrittenPOC = m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC;
m_sReoderingStatus.iLastWrittenSeqNum = m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iSeqNum;
memcpy(pDstInfo, &m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].sBufferInfo, sizeof(SBufferInfo));
ppDst[0] = pDstInfo->pDst[0];
ppDst[1] = pDstInfo->pDst[1];
Expand All @@ -1187,10 +1122,8 @@ void CWelsDecoder::ReleaseBufferedReadyPictureNoReorder(PWelsDecoderContext pCtx
PPicBuff pPicBuff = pCtx ? pCtx->pPicBuff : m_pPicBuff;
PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
--pPic->iRefCount;
}
if (m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP) {
--m_sReoderingStatus.iLastGOPRemainPicts;
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
if (pPic->iRefCount <= 0 && pPic->pSetUnRef)
pPic->pSetUnRef(pPic);
}
--m_sReoderingStatus.iNumOfPicts;
}
Expand All @@ -1204,8 +1137,12 @@ DECODING_STATE CWelsDecoder::ReorderPicturesInDisplay(PWelsDecoderContext pDecCo
m_bIsBaseline = pDecContext->pSps->uiProfileIdc == 66 || pDecContext->pSps->uiProfileIdc == 83;
if (!m_bIsBaseline) {
if (pDstInfo->iBufferStatus == 1) {
if (m_sReoderingStatus.iLastGOPRemainPicts == 0 && pDecContext->pSliceHeader->eSliceType == B_SLICE &&
pDecContext->pSliceHeader->iPicOrderCntLsb <= m_sReoderingStatus.iLastWrittenPOC + 2) {
if (pDecContext->pSliceHeader->eSliceType == B_SLICE &&
((pDecContext->iSeqNum == m_sReoderingStatus.iLastWrittenSeqNum) ?
(pDecContext->pSliceHeader->iPicOrderCntLsb <= m_sReoderingStatus.iLastWrittenPOC + 2) :
(pDecContext->iSeqNum - m_sReoderingStatus.iLastWrittenSeqNum == 1 && pDecContext->pSliceHeader->iPicOrderCntLsb == 0))) {
m_sReoderingStatus.iLastWrittenPOC = pDecContext->pSliceHeader->iPicOrderCntLsb;
m_sReoderingStatus.iLastWrittenSeqNum = pDecContext->iSeqNum;
//issue #3478, use b-slice type to determine correct picture order as the first priority as POC order is not as reliable as based on b-slice
ppDst[0] = pDstInfo->pDst[0];
ppDst[1] = pDstInfo->pDst[1];
Expand Down

0 comments on commit 4f01c15

Please sign in to comment.