2021-03-26

Hi3559AV100 NNIE开发(6)RFCN中NNIE实现关键线程函数->SAMPLE_SVP_NNIE_Rfcn_ViToVo()进行数据流分析

  前面随笔给出了NNIE开发的基本知识,下面几篇随笔将着重于Mobilefacenet NNIE开发,实现mobilefacenet.wk的chip版本,并在Hi3559AV100上实现mobilefacenet网络功能,外接USB摄像头通过MPP平台输出至VO HDMI显示结果。下文是Hi3559AV100 NNIE开发(6)RFCN中实现关键线程函数->SAMPLE_SVP_NNIE_Rfcn_ViToVo()进行数据流分析,通过对线程函数分析,详细了解如何对.wk模型数据进行处理并弄清楚检测框绘制这些后处理的实现。

1、SAMPLE_SVP_NNIE_Rfcn_ViToVo()函数调用

  首先给出SAMPLE_SVP_NNIE_Rfcn_ViToVo()函数的调用,为后续分析提供参照:

 1 static pthread_t s_hNnieThread = 0; //全局定义 2  3 HI_CHAR acThreadName[16] = {0};//局部定义 4  5  6 /****************************************** 7   Create work thread 8  ******************************************/ 9 snprintf(acThreadName, 16, "NNIE_ViToVo");10 prctl(PR_SET_NAME, (unsigned long)acThreadName, 0,0,0);11 pthread_create(&s_hNnieThread, 0, SAMPLE_SVP_NNIE_Rfcn_ViToVo, NULL); 

  其中prctl函数的定义如下,其中PR_SET_NAME表示使用(char *) arg2所指向的位置中的值设置调用线程的名称。

1 int prctl(int option, unsigned long arg2, unsigned long arg3,2     unsigned long arg4, unsigned long arg5);

2、SAMPLE_SVP_NNIE_Rfcn_ViToVo()函数实现具体分析

  下面讲分析SAMPLE_SVP_NNIE_Rfcn_ViToVo(HI_VOID* pArgs)函数的内部实现,具体设计到5个函数,实现的功能已经注释出来,具体如下:

 1 static HI_VOID* SAMPLE_SVP_NNIE_Rfcn_ViToVo(HI_VOID* pArgs) 2 { 3  4  .... //参数定义 5  6  while (HI_FALSE == s_bNnieStopSignal) 7  {  8   //用户从通道获取一帧处理完成的图像 通道1-输出stExtFrmInfo 9   s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp,10           as32VpssChn[1],11           &stExtFrmInfo,12            s32MilliSec);13   ......14 15   //用户从通道获取一帧处理完成的图像 通道0-输出stBaseFrmInfo16   s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 17           as32VpssChn[0], 18           &stBaseFrmInfo, 19            s32MilliSec);20   ......21 22   //关键处理函数23   s32Ret = SAMPLE_SVP_NNIE_Rfcn_Proc(24       pstParam,25       pstSwParam, 26      &stExtFrmInfo,27       stBaseFrmInfo.stVFrame.u32Width,28       stBaseFrmInfo.stVFrame.u32Height);29   ......30 31   //Draw rect32   s32Ret = SAMPLE_COMM_SVP_NNIE_FillRect(33      &stBaseFrmInfo, 34      &(pstSwParam->stRect), 35      0x0000FF00); //绿色36  ...... 37 38   //将视频图像送入指定输出通道显示。39   s32Ret = HI_MPI_VO_SendFrame(voLayer, 40           voChn, 41          &stBaseFrmInfo, 42           s32MilliSec);43  ......44  45 BASE_RELEASE:46   s32Ret = HI_MPI_VPSS_ReleaseChnFrame(s32VpssGrp,as32VpssChn[0], &stBaseFrmInfo);47  ......48 49 EXT_RELEASE:50   s32Ret = HI_MPI_VPSS_ReleaseChnFrame(s32VpssGrp,as32VpssChn[1], &stExtFrmInfo);51  .......52 53  }54 55  return HI_NULL;56 }

2.1、HI_MPI_VPSS_GetChnFrame的实现

  下面给出HI_MPI_VPSS_GetChnFrame函数的参数调用与实现过程(其中VPSS双通道输出),具体功能为从通道获取一帧处理完成的图像:

 1  HI_S32 s32VpssGrp = 0; 2  HI_S32 as32VpssChn[] = {VPSS_CHN0, VPSS_CHN1}; 3  VIDEO_FRAME_INFO_S stBaseFrmInfo; 4  VIDEO_FRAME_INFO_S stExtFrmInfo; 5  HI_S32 s32MilliSec = 20000; 6  7   //用户从通道获取一帧处理完成的图像 通道1-输出stExtFrmInfo 8  s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 9         as32VpssChn[1],10         &stExtFrmInfo,11         s32MilliSec);12 13   //用户从通道获取一帧处理完成的图像 通道0-输出stBaseFrmInfo14  s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 15         as32VpssChn[0], 16         &stBaseFrmInfo, 17         s32MilliSec);18 19  //函数定义20  HI_S32 HI_MPI_VPSS_GetChnFrame(VPSS_GRP VpssGrp, VPSS_CHN VpssChn,21 VIDEO_FRAME_INFO_S *pstVideoFrame, HI_S32 s32MilliSec);

2.2、SAMPLE_SVP_NNIE_Rfcn_Proc的实现-NNIE数据处理

  SAMPLE_SVP_NNIE_Rfcn_Proc函数实现是整个RFCN NNIE数据处理过程的Key Point,检测加框等信息来源处,现给出函数参数及实现分析:

 1  SAMPLE_SVP_NNIE_PARAM_S *pstParam; 2  SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam; 3  4  pstParam = &s_stRfcnNnieParam; 5  pstSwParam = &s_stRfcnSoftwareParam; 6 /*  7  s_stRfcnNnieParam及s_stRfcnSoftwareParam参数涉及前述操作 8  9  s_stRfcnNnieParam.pstModel = &s_stRfcnModel.stModel;10  s_stRfcnSoftwareParam.apcRpnDataLayerName[0] = "rpn_cls_score";11  s_stRfcnSoftwareParam.apcRpnDataLayerName[1] = "rpn_bbox_pred";12  s32Ret = SAMPLE_SVP_NNIE_Rfcn_ParamInit(&stNnieCfg,13          &s_stRfcnNnieParam,14           &s_stRfcnSoftwareParam);15 */16 17 18  VIDEO_FRAME_INFO_S stBaseFrmInfo;19  VIDEO_FRAME_INFO_S stExtFrmInfo;20 21 //其中stBaseFrmInfo参数与stExtFrmInfo参数是经过下述函数输出得到22 /*23  s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp,24         as32VpssChn[1],25        &stExtFrmInfo,26        s32MilliSec);27  s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 28         as32VpssChn[0], 29        &stBaseFrmInfo, 30         s32MilliSec);31 */32 33 34  s32Ret = SAMPLE_SVP_NNIE_Rfcn_Proc(35       pstParam,36       pstSwParam, 37      &stExtFrmInfo,38       stBaseFrmInfo.stVFrame.u32Width,39       stBaseFrmInfo.stVFrame.u32Height); 

  随之给出SAMPLE_SVP_NNIE_Rfcn_Proc函数的定义及内部实现过程:

 1 static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 2  SAMPLE_SVP_NNIE_PARAM_S *pstParam, 3  SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam,  4  VIDEO_FRAME_INFO_S* pstExtFrmInfo, 5  HI_U32 u32BaseWidth,HI_U32 u32BaseHeight) 6 { 7  8  ......参数定义 9 10  s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,11          &stInputDataIdx,12          &stProcSegIdx,13           HI_TRUE);14  15  ......16 17  /*RPN*/18  s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 19           pstSwParam);20  21 22  if(0 != pstSwParam->stRpnBbox.unShape.stWhc.u32Height)23  {24 25   s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(26      pstParam,27      &stInputDataIdx,28      &pstSwParam->stRpnBbox,29      &stProcSegIdx,30      HI_TRUE);31   ......32   33   s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(34      pstParam,35      &stInputDataIdx,36      &pstSwParam->stRpnBbox,37      &stProcSegIdx,38      HI_TRUE);39  ......40 41   s32Ret = SAMPLE_SVP_NNIE_Rfcn_GetResult(pstParam,42          pstSwParam);43   44  }45  else46  { ...... }47  s32Ret = SAMPLE_SVP_NNIE_RoiToRect(48     &(pstSwParam->stDstScore),49     &(pstSwParam->stDstRoi), 50     &(pstSwParam->stClassRoiNum), 51     pstSwParam->af32ScoreThr,52     HI_TRUE,53     &(pstSwParam->stRect),54     pstExtFrmInfo->stVFrame.u32Width, 55     pstExtFrmInfo->stVFrame.u32Height,56     u32BaseWidth,57     u32BaseHeight);58  59  ......60 61  return s32Ret;62 63 }

2.2.1、SAMPLE_SVP_NNIE_Forward子函数分析

   首先给出函数的调用及参数细节,便于分析函数功能:

 1  其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函数中: 2  SAMPLE_SVP_NNIE_PARAM_S *pstParam; 3  pstParam = &s_stRfcnNnieParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc 4   5  6  7  static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 8   SAMPLE_SVP_NNIE_PARAM_S *pstParam,  /*这个参数传进来之后经过了SP420赋值然后送入 9             SAMPLE_SVP_NNIE_Forward*/10   SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, 11   VIDEO_FRAME_INFO_S* pstExtFrmInfo,12   HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)13 14  ......15 16  SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S stInputDataIdx = {0};17  SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S stProcSegIdx = {0};18 19  stInputDataIdx.u32SegIdx = 0;20  stInputDataIdx.u32NodeIdx = 0;21 22  /*SP420*/23  pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0];24  pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0];25  pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride = pstExtFrmInfo->stVFrame.u32Stride[0];26 27  /*NNIE process 0-th seg*/28  stProcSegIdx.u32SegIdx = 0;29 30  /*NNIE process 0-th seg*/31  stProcSegIdx.u32SegIdx = 0;32  s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,33          &stInputDataIdx,34          &stProcSegIdx,35           HI_TRUE);36 37  ......38 39  //函数定义40  static HI_S32 SAMPLE_SVP_NNIE_Forward(41     SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam,42     SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx,43     SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx,44     HI_BOOL bInstant)

  此函数如其名,主要实现了NNIE forward功能,下面给出调用具体函数:

 1 /****************************************************************************** 2 * function : NNIE Forward 3 ******************************************************************************/ 4 static HI_S32 SAMPLE_SVP_NNIE_Forward(SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam, 5  SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx, 6  SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx,HI_BOOL bInstant) 7 { 8  ...... 9 10  SAMPLE_COMM_SVP_FlushCache(11  pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64PhyAddr,12  (HI_VOID *) pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64VirAddr,13  pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u32Size);14 15  /*set input blob according to node name*/16  if(pstInputDataIdx->u32SegIdx != pstProcSegIdx->u32SegIdx)17  {18   for(i = 0; i < pstNnieParam->pstModel->astSeg[pstProcSegIdx->u32SegIdx].u16SrcNum; i++)19   {20    ......21   }22  }23 24  /*NNIE_Forward 多节点输入输出的 CNN 类型网络预测。 25  对输入样本(s)进行CNN预测,对对应样本(s)进行输出响应*/26  s32Ret = HI_MPI_SVP_NNIE_Forward(27   &hSvpNnieHandle,28   pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astSrc,29   pstNnieParam->pstModel, 30   pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst,31   &pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx], 32   bInstant);33  34 35  if(bInstant)36  {37   /*Wait NNIE finish,,,,enNnieId 执行网络段的 NNIE 引擎 ID。 */38   while(HI_ERR_SVP_NNIE_QUERY_TIMEOUT == (s32Ret = HI_MPI_SVP_NNIE_Query(pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].enNnieId,39    hSvpNnieHandle, &bFinish, HI_TRUE)))40   {41    ......42   }43  }44 45  bFinish = HI_FALSE;46  for(i = 0; i < pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].u32DstNum; i++)47  {48   if(SVP_BLOB_TYPE_SEQ_S32 == pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].enType)49   {50    ......51    52    SAMPLE_COMM_SVP_FlushCache(53  pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,54  (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,55  u32TotalStepNum*pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);56 57   }58   else59   {60 61    SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,62     (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,63     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Num*64     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Chn*65     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Height*66     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);67   }68  }69 70  return s32Ret;71 }

2.2.2、SAMPLE_SVP_NNIE_Rfcn_Rpn子函数分析

   下面给出sample_svp_NNIE_Rfcn_Rpn子函数分析,此函数主要是用于rpn相关,下面给出参数调用及函数分析:

 1  其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函数中: 2  SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc 3  pstParam = &s_stRfcnNnieParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc 4  5  SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam; 6  pstSwParam = &s_stRfcnSoftwareParam; 7  8  static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 9   SAMPLE_SVP_NNIE_PARAM_S *pstParam,  /*这个参数传进来之后经过了SP420赋值然后送入10             SAMPLE_SVP_NNIE_Forward*/11   SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, //直接传给SAMPLE_SVP_NNIE_Rfcn_Rpn函数12   VIDEO_FRAME_INFO_S* pstExtFrmInfo,13   HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)14 15  /*SP420*/16  pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0];17  pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0];18  pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride = pstExtFrmInfo->stVFrame.u32Stride[0];19 20  此后,pstParam参数先传入SAMPLE_SVP_NNIE_Forward函数进行处理,随后传入SAMPLE_SVP_NNIE_Rfcn_Rpn函数21 22   /*RPN*/23  s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 24           pstSwParam);25 26  // 函数调用 ,用于used to do rpn27  HI_S32 SAMPLE_SVP_NNIE_Rfcn_Rpn(28   SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,29   SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)

  下面给出sample_svp_NNIE_Rfcn_Rpn子函数内部实现,此函数虽然传入两个参数,但是没有对pstNnieParam参数进行任何处理,完成的pstSoftwareParam结构体内大量参数的的处理 ,主要是调用了SVP_NNIE_Rpn函数与SAMPLE_COMM_SVP_FlushCache函数:

 1 HI_S32 SAMPLE_SVP_NNIE_Rfcn_Rpn(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam, 2  SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam) 3 { 4  HI_S32 s32Ret = HI_SUCCESS; 5  s32Ret = SVP_NNIE_Rpn(pstSoftwareParam->aps32Conv,pstSoftwareParam->u32NumRatioAnchors, 6   pstSoftwareParam->u32NumScaleAnchors,pstSoftwareParam->au32Scales, 7   pstSoftwareParam->au32Ratios,pstSoftwareParam->u32OriImHeight, 8   pstSoftwareParam->u32OriImWidth,pstSoftwareParam->au32ConvHeight, 9   pstSoftwareParam->au32ConvWidth,pstSoftwareParam->au32ConvChannel,10   pstSoftwareParam->u32ConvStride,pstSoftwareParam->u32MaxRoiNum,11   pstSoftwareParam->u32MinSize,pstSoftwareParam->u32SpatialScale,12   pstSoftwareParam->u32NmsThresh,pstSoftwareParam->u32FilterThresh,13   pstSoftwareParam->u32NumBeforeNms,(HI_U32*)pstSoftwareParam->stRpnTmpBuf.u64VirAddr,14   (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr,15   &pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height);16  SAMPLE_COMM_SVP_FlushCache(pstSoftwareParam->stRpnBbox.u64PhyAddr,17   (HI_VOID *) pstSoftwareParam->stRpnBbox.u64VirAddr,18   pstSoftwareParam->stRpnBbox.u32Num*19   pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Chn*20   pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height*21   pstSoftwareParam->stRpnBbox.u32Stride);22  SAMPLE_SVP_CHECK_EXPR_RET(HI_SUCCESS != s32Ret,s32Ret,SAMPLE_SVP_ERR_LEVEL_ERROR,23   "Error,SVP_NNIE_Rpn failed!\n");24  return s32Ret;25 }

2.2.3、SAMPLE_SVP_NNIE_ForwardWithBbox子函数分析

  完成前面几个函数后,通过if判断,当满足条件后执行SAMPLE_SVP_NNIE_ForwardWithBbox函数:

1 if(0 != pstSwParam->stRpnBbox.unShape.stWhc.u32Height)

  否则不满足if条件的时候,将执行下面赋值语句:

1   for (i = 0; i < pstSwParam->stClassRoiNum.unShape.stWhc.u32Width; i++)2   {3    *(((HI_U32*)(HI_UL)pstSwParam->stClassRoiNum.u64VirAddr)+i) = 0;4   }

  而SAMPLE_SVP_NNIE_ForwardWithBbox执行了有两次,分别是对不同NNIE process x-th seg进行处理,具体如下:

 1  其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函数中: 2  SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc 3  pstParam = &s_stRfcnNnieParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc 4  5  SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam; 6  pstSwParam = &s_stRfcnSoftwareParam; 7  8  static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 9   SAMPLE_SVP_NNIE_PARAM_S *pstParam,  /*这个参数传进来之后经过了SP420赋值然后送入10             SAMPLE_SVP_NNIE_Forward*/11   SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, /*直接传给SAMPLE_SVP_NNIE_Rfcn_Rpn函数12                随后传给SAMPLE_SVP_NNIE_ForwardWithBbox函数*/13   VIDEO_FRAME_INFO_S* pstExtFrmInfo,14   HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)15 16  /*RPN*/17  s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 18          pstSwParam); /*此函数完成pstSwParam的相关赋值,并19               传给SAMPLE_SVP_NNIE_ForwardWithBbox函数*/20 21 22  /*在SAMPLE_SVP_NNIE_ForwardWithBbox函数前面,已经有了下面参数的赋值,23  并且传入至SAMPLE_SVP_NNIE_Forward函数*/24   stInputDataIdx.u32SegIdx = 0;25   stInputDataIdx.u32NodeIdx = 0;26 27  s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,28          &stInputDataIdx, //stInputDataIdx参数已经用于此函数中29          &stProcSegIdx,30           HI_TRUE);31 32 33  //函数调用34  /*NNIE process 1-th seg, the input data comes from 3-rd report node of 0-th seg,35   the input roi comes from RPN results*/36  stInputDataIdx.u32SegIdx = 0;37  stInputDataIdx.u32NodeIdx = 3;38 39  stProcSegIdx.u32SegIdx = 1;40 41  s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(42     pstParam,43     &stInputDataIdx,44     &pstSwParam->stRpnBbox,45     &stProcSegIdx,46     HI_TRUE);47 48 49  /*NNIE process 2-nd seg, the input data comes from 4-th report node of 0-th seg50   the input roi comes from RPN results*/51  stInputDataIdx.u32SegIdx = 0;52  stInputDataIdx.u32NodeIdx = 4;53 54  stProcSegIdx.u32SegIdx = 2;55 56  s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(57     pstParam,58     &stInputDataIdx,59     &pstSwParam->stRpnBbox,60     &stProcSegIdx,61     HI_TRUE);

  下面给出函数具体实现:

 1 /****************************************************************************** 2 * function : NNIE ForwardWithBbox 3 ******************************************************************************/ 4 static HI_S32 SAMPLE_SVP_NNIE_ForwardWithBbox( 5    SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam, 6    SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx, 7    SVP_SRC_BLOB_S astBbox[], 8    SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx, 9    HI_BOOL bInstant)10 {11  ......12 13  SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64PhyAddr,14   (HI_VOID *) pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64VirAddr,15   pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u32Size);16 17  /*set input blob according to node name*/18  if(pstInputDataIdx->u32SegIdx != pstProcSegIdx->u32SegIdx)19  {20   for(i = 0; i < pstNnieParam->pstModel->astSeg[pstProcSegIdx->u32SegIdx].u16SrcNum; i++)21   {22    for(j = 0; j < pstNnieParam->pstModel->astSeg[pstInputDataIdx->u32SegIdx].u16DstNum; j++)23    {24     ......25    }26    ......27   }28  }29  /*NNIE_ForwardWithBbox*/30  s32Ret = HI_MPI_SVP_NNIE_ForwardWithBbox(31   &hSvpNnieHandle,32   pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astSrc,33   astBbox,34   pstNnieParam->pstModel, //网络类型只支持ROI/PSROI35   pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst,36   &pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx], 37   bInstant);38 39  ......40 41  if(bInstant)42  {43   /*Wait NNIE finish*/44   while(HI_ERR_SVP_NNIE_QUERY_TIMEOUT == (s32Ret = HI_MPI_SVP_NNIE_Query(pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].enNnieId,45    hSvpNnieHandle, &bFinish, HI_TRUE)))46   {47    ......48   }49  }50 51  bFinish = HI_FALSE;52 53 54  for(i = 0; i < pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].u32DstNum; i++)55  {56   if(SVP_BLOB_TYPE_SEQ_S32 == pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].enType)57   {58    ......59    60    SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,61     (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,62     u32TotalStepNum*pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);63   }64   else65   {66    SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,67     (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,68     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Num*69     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Chn*70     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Height*71     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);72   }73  }74 75  return s32Ret;76 }

2.2.4、SAMPLE_SVP_NNIE_Rfcn_GetResult子函数分析

  下一个子函数是获得NNIE Rfcn的结果,前提需要保证网络结构和输入数据保持一致,函数调用如下:

 

 1  其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函数中: 2  SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc 3  pstParam = &s_stRfcnNnieParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc 4  5  SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam; 6  pstSwParam = &s_stRfcnSoftwareParam; 7  8  static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc( 9   SAMPLE_SVP_NNIE_PARAM_S *pstParam,  /*这个参数传进来之后经过了SP420赋值然后送入10             SAMPLE_SVP_NNIE_Forward*/11   SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, /*直接传给SAMPLE_SVP_NNIE_Rfcn_Rpn函数12                随后传给SAMPLE_SVP_NNIE_ForwardWithBbox函数*/13   VIDEO_FRAME_INFO_S* pstExtFrmInfo,14   HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)15 16  pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0];17  pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0];18  pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride = pstExtFrmInfo->stVFrame.u32Stride[0];19 20 21  s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam, //参数调用22          &stInputDataIdx,23          &stProcSegIdx,24           HI_TRUE);25  26  /*RPN*/27  s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, //参数调用28          pstSwParam); //参数调用29 30 31  s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(32     pstParam, //参数调用33     &stInputDataIdx,34     &pstSwParam->stRpnBbox,35     &stProcSegIdx,36     HI_TRUE);37 38 39 40  //函数调用41 42  /*GetResult*/43  /*if user has changed net struct, please make sure SAMPLE_SVP_NNIE_Rfcn_GetResult44   function's input datas are correct*/45  s32Ret = SAMPLE_SVP_NNIE_Rfcn_GetResult(pstParam,46            pstSwParam);47 48  //函数调用49  HI_S32 SAMPLE_SVP_NNIE_Rfcn_GetResult(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,50          SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)

  此函数作用是为了获取NNIE  RFCN的结果,其核心是实现了SVP_NNIE_Rfcn_GetResult函数,具体如下:

 1 HI_S32 SAMPLE_SVP_NNIE_Rfcn_GetResult(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam, 2          SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam) 3 { 4  HI_S32 s32Ret = HI_SUCCESS; 5  HI_U32 i = 0; 6  HI_S32* ps32Proposal = (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr; 7  8  ...... 9 10  for(i = 0; i < pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height; i++)11  {12   *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i) /= SAMPLE_SVP_NNIE_QUANT_BASE;13   *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+1) /= SAMPLE_SVP_NNIE_QUANT_BASE;14   *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+2) /= SAMPLE_SVP_NNIE_QUANT_BASE;15   *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+3) /= SAMPLE_SVP_NNIE_QUANT_BASE;16  }17  //this function is used to get RFCN result18  s32Ret = SVP_NNIE_Rfcn_GetResult(19     (HI_S32*)pstNnieParam->astSegData[1].astDst[0].u64VirAddr,20     pstNnieParam->astSegData[1].astDst[0].u32Stride,21     (HI_S32*)pstNnieParam->astSegData[2].astDst[0].u64VirAddr,22     pstNnieParam->astSegData[2].astDst[0].u32Stride,23     (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr,24     pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height,25     pstSoftwareParam->au32ConfThresh,pstSoftwareParam->u32MaxRoiNum,26     pstSoftwareParam->u32ClassNum,pstSoftwareParam->u32OriImWidth,27     pstSoftwareParam->u32OriImHeight,pstSoftwareParam->u32ValidNmsThresh,28     (HI_U32*)pstSoftwareParam->stGetResultTmpBuf.u64VirAddr,29     (HI_S32*)pstSoftwareParam->stDstScore.u64VirAddr,30     (HI_S32*)pstSoftwareParam->stDstRoi.u64VirAddr,31     (HI_S32*)pstSoftwareParam->stClassRoiNum.u64VirAddr);32  ......33  return s32Ret;34 }

2.2.5、SAMPLE_SVP_NNIE_RoiToRect子函数分析

  SAMPLE_SVP_NNIE_Rfcn_Proc函数最后一个子函数为对ROI(感兴趣区域画框),RFNC对21类物体进行目标识别,具体调用及实现如下:

 1  /*draw result, this sample has 21 classes: 2   class 0:background  class 1:plane   class 2:bicycle 3   class 3:bird   class 4:boat   class 5:bottle 4   class 6:bus   class 7:car    class 8:cat 5   class 9:chair   class10:cow    class11:diningtable 6   class 12:dog   class13:horse   class14:motorbike 7   class 15:person  class16:pottedplant  class17:sheep 8   class 18:sofa   class19:train   class20:tvmonitor*/ 9  s32Ret = SAMPLE_SVP_NNIE_RoiToRect(10     &(pstSwParam->stDstScore),11     &(pstSwParam->stDstRoi), 12     &(pstSwParam->stClassRoiNum), 13     pstSwParam->af32ScoreThr,14      HI_TRUE,15     &(pstSwParam->stRect),16     pstExtFrmInfo->stVFrame.u32Width, 17     pstExtFrmInfo->stVFrame.u32Height,18      u32BaseWidth,19      u32BaseHeight);20 21 /******************************************************************************22 * function : roi to rect23 ******************************************************************************/24 HI_S32 SAMPLE_SVP_NNIE_RoiToRect(SVP_BLOB_S *pstDstScore,25         SVP_BLOB_S *pstDstRoi, SVP_BLOB_S *pstClassRoiNum, HI_FLOAT *paf32ScoreThr,26         HI_BOOL bRmBg,SAMPLE_SVP_NNIE_RECT_ARRAY_S *pstRect,27          HI_U32 u32SrcWidth, HI_U32 u32SrcHeight,HI_U32 u32DstWidth,HI_U32 u32DstHeight)28 {29  HI_U32 i = 0, j = 0;30  HI_U32 u32RoiNumBias = 0;31  HI_U32 u32ScoreBias = 0;32  HI_U32 u32BboxBias = 0;33  HI_FLOAT f32Score = 0.0f;34  HI_S32* ps32Score = (HI_S32*)pstDstScore->u64VirAddr;35  HI_S32* ps32Roi = (HI_S32*)pstDstRoi->u64VirAddr;36  HI_S32* ps32ClassRoiNum = (HI_S32*)pstClassRoiNum->u64VirAddr;37  HI_U32 u32ClassNum = pstClassRoiNum->unShape.stWhc.u32Width;38  HI_U32 u32RoiNumTmp = 0;39 40  .......41  42  pstRect->u32TotalNum = 0;43  pstRect->u32ClsNum = u32ClassNum;44  if (bRmBg)45  {46   pstRect->au32RoiNum[0] = 0;47   u32RoiNumBias += ps32ClassRoiNum[0];48   for (i = 1; i < u32ClassNum; i++)49   {50    u32ScoreBias = u32RoiNumBias;51    u32BboxBias = u32RoiNumBias * SAMPLE_SVP_NNIE_COORDI_NUM;52    u32RoiNumTmp = 0;53    /*if the confidence score greater than result thresh, the result will be drawed*/54    if(((HI_FLOAT)ps32Score[u32ScoreBias] / SAMPLE_SVP_NNIE_QUANT_BASE >=55      paf32ScoreThr[i]) && (ps32ClassRoiNum[i] != 0))56    {57     for (j = 0; j < (HI_U32)ps32ClassRoiNum[i]; j++)58     {59      /*Score is descend order*/60      f32Score = (HI_FLOAT)ps32Score[u32ScoreBias + j] / SAMPLE_SVP_NNIE_QUANT_BASE;61      if ((f32Score < paf32ScoreThr[i]) || (u32RoiNumTmp >= SAMPLE_SVP_NNIE_MAX_ROI_NUM_OF_CLASS))62      {63       break;64      }65 66      pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32X = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM] / (HI_FLOAT)u32SrcWidth * (HI_FLOAT)u32DstWidth) & (~1) ;67      pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32Y = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 1] / (HI_FLOAT)u32SrcHeight * (HI_FLOAT)u32DstHeight) & (~1);68 69      pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32X = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 2]/ (HI_FLOAT)u32SrcWidth * (HI_FLOAT)u32DstWidth) & (~1);70      pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32Y = pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32Y;71 72      pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32X = pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32X;73      pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32Y = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 3] / (HI_FLOAT)u32SrcHeight * (HI_FLOAT)u32DstHeight) & (~1);74 75      pstRect->astRect[i][u32RoiNumTmp].astPoint[3].s32X = pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32X;76      pstRect->astRect[i][u32RoiNumTmp].astPoint[3].s32Y = pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32Y;77 78      u32RoiNumTmp++;79     }80 81    }82 83    pstRect->au32RoiNum[i] = u32RoiNumTmp;84    pstRect->u32TotalNum += u32RoiNumTmp;85    u32RoiNumBias += ps32ClassRoiNum[i];86   }87 88  }89  return HI_SUCCESS;90 }

3、SAMPLE_COMM_SVP_NNIE_FillRect函数分析

  SAMPLE_COMM_SVP_NNIE_FillRect主要是配合VGS实现画框功能,具体调用和实现如下,函数功能已由注释给出:

 1  //Draw rect 2  s32Ret = SAMPLE_COMM_SVP_NNIE_FillRect( 3     &stBaseFrmInfo,  4     &(pstSwParam->stRect),  5     0x0000FF00); //绿色 6  7 HI_S32 SAMPLE_COMM_SVP_NNIE_FillRect( 8  VIDEO_FRAME_INFO_S *pstFrmInfo, 9  SAMPLE_SVP_NNIE_RECT_ARRAY_S* pstRect, 10  HI_U32 u32Color)11 {12  VGS_HANDLE VgsHandle = -1;13  HI_S32 s32Ret = HI_SUCCESS;14  HI_U32 i,j;15  VGS_TASK_ATTR_S stVgsTask;//定义 VGS task 的属性16  VGS_ADD_COVER_S stVgsAddCover;//定义 VGS 上 COVER 的配置17  static HI_U32 u32Frm = 0;18  u32Frm++;19  if (0 == pstRect->u32TotalNum)20  {21   return s32Ret;22  }23  s32Ret = HI_MPI_VGS_BeginJob(&VgsHandle); //启动一个 job。24  if (s32Ret != HI_SUCCESS)25  {26   ......27   return s32Ret;28  }29 30  memcpy(&stVgsTask.stImgIn, pstFrmInfo, sizeof(VIDEO_FRAME_INFO_S));31  memcpy(&stVgsTask.stImgOut, pstFrmInfo, sizeof(VIDEO_FRAME_INFO_S));32 33  stVgsAddCover.enCoverType = COVER_QUAD_RANGLE;//任意四边形COVER34  stVgsAddCover.u32Color = u32Color; //RGB88835  stVgsAddCover.stQuadRangle.bSolid = HI_FALSE; //空心 COVER36  stVgsAddCover.stQuadRangle.u32Thick = 2; //2 像素对齐37 38  for (i = 0; i < pstRect->u32ClsNum; i++)39  {40   for (j = 0; j < pstRect->au32RoiNum[i]; j++)41   {42    memcpy(stVgsAddCover.stQuadRangle.stPoint, pstRect->astRect[i][j].astPoint, sizeof(pstRect->astRect[i][j].astPoint));43    44    //做 COVER 任务时,输入输出图像为同一块 buffer45    //往一个已经启动的 job 里添加打 COVER task。 task属性必须满足VGS的能力。46    s32Ret = HI_MPI_VGS_AddCoverTask(VgsHandle, &stVgsTask, &stVgsAddCover);47    if (s32Ret != HI_SUCCESS)48    {49     SAMPLE_PRT("HI_MPI_VGS_AddCoverTask fail,Error(%#x)\n", s32Ret);50     HI_MPI_VGS_CancelJob(VgsHandle);51     return s32Ret;52    }53 54   }55 56  }57  //提交一个 job。58  s32Ret = HI_MPI_VGS_EndJob(VgsHandle);59  if (s32Ret != HI_SUCCESS)60  {61   SAMPLE_PRT("HI_MPI_VGS_EndJob fail,Error(%#x)\n", s32Ret);62   HI_MPI_VGS_CancelJob(VgsHandle);63   return s32Ret;64  }65 66  return s32Ret;

4、HI_MPI_VO_SendFrame函数分析

  SAMPLE_SVP_NNIE_Rfcn_ViToVo线程函数执行的最后一个函数HI_MPI_VO_SendFrame,函数作用是将视频图像送入指定输出通道显示,具体调用如下:

1   s32Ret = HI_MPI_VO_SendFrame(voLayer, 2           voChn, 3          &stBaseFrmInfo, 4          s32MilliSec);

 

  

 









原文转载:http://www.shaoqun.com/a/642740.html

跨境电商:https://www.ikjzd.com/

刘军:https://www.ikjzd.com/w/1835

拍拍网:https://www.ikjzd.com/w/2205


前面随笔给出了NNIE开发的基本知识,下面几篇随笔将着重于MobilefacenetNNIE开发,实现mobilefacenet.wk的chip版本,并在Hi3559AV100上实现mobilefacenet网络功能,外接USB摄像头通过MPP平台输出至VOHDMI显示结果。下文是Hi3559AV100NNIE开发(6)RFCN中实现关键线程函数->SAMPLE_SVP_NNIE_Rfcn_
ebay易趣:https://www.ikjzd.com/w/210
tenso:https://www.ikjzd.com/w/1552
好东东网:https://www.ikjzd.com/w/1238
重大突发!2艘满载中国货物集装箱船严重碰撞!:https://www.ikjzd.com/home/17883
2017跨境电商选品大会:https://www.kjyunke.com/courses/375
祸不单行!亚马逊仓库着火,欧洲仓拒收大件商品!:https://www.ikjzd.com/home/118509

No comments:

Post a Comment