本節內容主要分為3部分,第一部分是流程結構圖;第二部分為人臉識別代碼流程;第三部分為具體的代碼分析。
1.流程結構圖
2.人臉識別代碼流程
1、人臉數據的初始化:
init_all_rockx_face_data();init_face_data();
2、創建rtsp會話,這里包括發送碼流數據,得客戶端,也就是我們在windows上用ffplay去拉流得時候,才會發送碼流數據給客戶端:
create_rtsp_demo(554);
rtsp_new_session
rtsp_set_video
rtsp_sync_video_ts
**
3、初始化vi通道屬性**
VI_CHN_ATTR_S vi_chn_attr;
4、初始化視頻處理屬性
RGA_ATTR_S stRgaAttr;
5、初始化編碼通道屬性:
VENC_CHN_ATTR_S venc_chn_attr;
6、綁定數據源:
RK_MPI_SYS_Bind
7、開始捕獲碼流:
RK_MPI_VI_StartStream
8、執行三個對應的線程:
- pthread_create(&rockx_pid, NULL, rockx_vi_detect_thread, NULL);
- pthread_create(&venc_pid, NULL, rockx_vi_face_recognize_venc_thread, NULL);
- pthread_create(&rtsp_pid, NULL, rockx_venc_rtsp_thread, NULL);
**9、銷毀申請的系統資源; **
3.核心代碼分析
初始化vi通道屬性;初始化視頻處理屬性;初始化編碼通道屬性;綁定數據源;開始捕獲碼流:
//初始化vi通道屬性VI_CHN_ATTR_S vi_chn_attr;vi_chn_attr.pcVideoNode = pDeviceName;vi_chn_attr.u32BufCnt = u32BufCnt;vi_chn_attr.u32Width = u32Width;vi_chn_attr.u32Height = u32Height;//你的攝像頭分辨率大小不要超過vencvi_chn_attr.enPixFmt = IMAGE_TYPE_NV12;vi_chn_attr.enBufType = VI_CHN_BUF_TYPE_MMAP;vi_chn_attr.enWorkMode = VI_WORK_MODE_NORMAL;ret = RK_MPI_VI_SetChnAttr(s32CamId, 0, &vi_chn_attr);//設置vi通道屬性ret |= RK_MPI_VI_EnableChn(s32CamId, 0);//使能vi通道屬性,讓其生效if (ret){printf("ERROR: create rkisp0 VI[0] error! ret=%d\n", ret);return 0;}//初始化rga屬性RGA_ATTR_S stRgaAttr;stRgaAttr.bEnBufPool = RK_TRUE;stRgaAttr.u16BufPoolCnt = 2;stRgaAttr.u16Rotaion = 0;stRgaAttr.stImgIn.u32X = 0;stRgaAttr.stImgIn.u32Y = 0;stRgaAttr.stImgIn.imgType = IMAGE_TYPE_NV12;stRgaAttr.stImgIn.u32Width = u32Width;stRgaAttr.stImgIn.u32Height = u32Height;stRgaAttr.stImgIn.u32HorStride = u32Width;stRgaAttr.stImgIn.u32VirStride = u32Height;stRgaAttr.stImgOut.u32X = 0;stRgaAttr.stImgOut.u32Y = 0;stRgaAttr.stImgOut.imgType = IMAGE_TYPE_NV12;stRgaAttr.stImgOut.u32Width = disp_width;stRgaAttr.stImgOut.u32Height = disp_height;stRgaAttr.stImgOut.u32HorStride = disp_width;stRgaAttr.stImgOut.u32VirStride = disp_height;ret = RK_MPI_RGA_CreateChn(0, &stRgaAttr);//rga通道if (ret){printf("ERROR: Create rga[0] falied! ret=%d\n", ret);return -1;}//初始化編碼屬性VENC_CHN_ATTR_S venc_chn_attr;memset(&venc_chn_attr, 0, sizeof(VENC_CHN_ATTR_S));venc_chn_attr.stVencAttr.u32PicWidth = disp_width;venc_chn_attr.stVencAttr.u32PicHeight = disp_height;venc_chn_attr.stVencAttr.u32VirWidth = disp_width;venc_chn_attr.stVencAttr.u32VirHeight = disp_height;venc_chn_attr.stVencAttr.imageType = IMAGE_TYPE_NV12;venc_chn_attr.stVencAttr.enType = RK_CODEC_TYPE_H264;venc_chn_attr.stVencAttr.u32Profile = 66;venc_chn_attr.stRcAttr.enRcMode = VENC_RC_MODE_H264CBR;//恒定的編碼碼率類型venc_chn_attr.stRcAttr.stH264Cbr.u32Gop = 30;venc_chn_attr.stRcAttr.stH264Cbr.u32BitRate = disp_width * disp_height * 3;venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateDen = 1;venc_chn_attr.stRcAttr.stH264Cbr.fr32DstFrameRateNum = 25;venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateDen = 1;venc_chn_attr.stRcAttr.stH264Cbr.u32SrcFrameRateNum = 25;ret = RK_MPI_VENC_CreateChn(0, &venc_chn_attr);//創建編碼通道if (ret){printf("ERROR: Create venc failed!\n");exit(0);}//初始化mpp通道MPP_CHN_S vi_chn;MPP_CHN_S rga_chn;vi_chn.enModId = RK_ID_VI;vi_chn.s32ChnId = 0;rga_chn.enModId = RK_ID_RGA;rga_chn.s32ChnId = 0;ret = RK_MPI_SYS_Bind(&vi_chn, &rga_chn);//綁定vi和rga通道if (ret != 0){printf("[VI] vi id: %d bind venc id: %d, ret: %d error\n", vi_chn.s32ChnId, rga_chn.s32ChnId, ret);return -1;}
初始化三個線程
//初始化三個線程idpthread_t rockx_pid;pthread_t venc_pid;pthread_t rtsp_pid;//創建人臉檢測線程pthread_create(&rockx_pid, NULL, rockx_vi_detect_thread, NULL);//人臉識別線程pthread_create(&venc_pid, NULL, rockx_vi_face_recognize_venc_thread, NULL);//人臉編碼rtsp傳輸線程pthread_create(&rtsp_pid, NULL, rockx_venc_rtsp_thread, NULL);
人臉檢測線程
void *rockx_vi_detect_thread(void *args)
{//自動釋放線程資源pthread_detach(pthread_self());//創建一個類對象thread_mapS_THREAD_MAP thread_map;//對thread_map進行初始化get_thread_map(0, &thread_map);//定義了一個map類型database_face_map對象map<string, rockx_face_feature_t> database_face_map = thread_map.thread_map;//定義迭代器database_itermap<string, rockx_face_feature_t>::iterator database_iter;//定義一個緩沖區MEDIA_BUFFER src_mb = NULL;//人臉模式枚舉變量定義rockx_module_t data_version;data_version = ROCKX_MODULE_FACE_DETECTION_V2;//定義一個人臉執行返回結果變量rockx_ret_t rockx_ret;//定義人臉檢測處理指針變量rockx_handle_t face_det_handle;//定義人臉識別特征提取處理指針變量rockx_handle_t face_recognize_handle;//定義人臉特征點定位處理指針變量rockx_handle_t face_5landmarks_handle;//定義了人臉標記檢測處理指針變量rockx_handle_t face_masks_det_handle;//定義人臉配置結構體指針變量rockx_config_t *config = rockx_create_config();//獲取人臉配置值//添加配置人臉模型存放路徑,這里是存放在共享目錄下:/mnt/nfs/rockx_data/rockx_add_config(config, ROCKX_CONFIG_DATA_PATH, "/mnt/nfs/rockx_data/");//創建使用人臉模型數據,來處理人臉檢測rockx_ret = rockx_create(&face_det_handle, data_version, config,sizeof(rockx_config_t));//判斷是否創建使用人臉模型數據 來處理人臉檢測是否成功 if (rockx_ret != ROCKX_RET_SUCCESS){printf("init face_detect error %d\n", rockx_ret);return NULL;}//使用人臉模型數據來人臉識別特征提取rockx_ret = rockx_create(&face_recognize_handle, ROCKX_MODULE_FACE_RECOGNIZE,config, sizeof(rockx_config_t));//識別是否成功 if (rockx_ret != ROCKX_RET_SUCCESS){printf("init face_recognize error %d\n", rockx_ret);return NULL;}//使用模型算法數據來做人臉特征點定位處理rockx_ret = rockx_create(&face_5landmarks_handle,ROCKX_MODULE_FACE_LANDMARK_5, config, 0);//判斷是否處理成功 if (rockx_ret != ROCKX_RET_SUCCESS){printf("init rockx module ROCKX_MODULE_FACE_LANDMARK_68 error %d\n",rockx_ret);}// rockx_handle_t face_masks_det_handle;進行標記處理rockx_ret = rockx_create(&face_masks_det_handle,ROCKX_MODULE_FACE_MASKS_DETECTION, config, 0);if (rockx_ret != ROCKX_RET_SUCCESS){printf("init rockx module ROCKX_MODULE_FACE_MASKS_DETECTION error %d\n",rockx_ret);}//定義人臉圖片結構體變量,并進行成員賦值rockx_image_t input_image;input_image.width = 1920;input_image.height = 1080;input_image.pixel_format = ROCKX_PIXEL_FORMAT_YUV420SP_NV12;bool is_recognize = false;string predict = "";//rockx_face_result_t face_result;int ret;//做輪詢操作while (!quit){
#if 1//從指定通道中獲取數據緩沖區src_mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, 0, -1);if (!src_mb){printf("ERROR: RK_MPI_SYS_GetMediaBuffer get null buffer!\n");break;}//從指定的MEDIA_BUFFER中獲取緩沖區數據大小input_image.size = RK_MPI_MB_GetSize(src_mb);input_image.data = (unsigned char *)RK_MPI_MB_GetPtr(src_mb);//從指定的MEDIA_BUFFER中獲取緩沖區數據指針
#endif#if 1//rockx_face_result_group_t face_result_group;//memset(&face_result_group, 0, sizeof(face_result_group));//定義人臉處理結果結構體變量rockx_object_array_t face_array;memset(&face_array, 0, sizeof(face_array));//開始人臉檢測rockx_ret = rockx_face_detect(face_det_handle, &input_image, &face_array, NULL);if (rockx_ret != ROCKX_RET_SUCCESS){printf("rockx_face_detect ERROR %d\n", rockx_ret);}//進行互斥處理set_rockx_face_array(face_array);//判斷檢測人臉特征數量值是否大于0if (face_array.count > 0){//rockx_queue->putRockxFaceArray(face_array);printf("face_count : %d\n", face_array.count);for (int i = 0; i < face_array.count; i++){if (1){int is_false_face;//進行人臉過濾處理ret = rockx_face_filter(face_5landmarks_handle, &input_image,&face_array.object[i].box, &is_false_face);if (ret != ROCKX_RET_SUCCESS){printf("rockx_face_filter error %d\n", ret);}if (is_false_face)continue;}
#if 1//人臉檢測結果(包括人臉、車牌、頭部、物體等)變量定義rockx_object_t max_face;rockx_object_t cur_face = face_array.object[i];//進行人臉區域計算處理操作int cur_face_box_area = (cur_face.box.right - cur_face.box.left) *(cur_face.box.bottom - cur_face.box.top);int max_face_box_area = (max_face.box.right - max_face.box.left) *(max_face.box.bottom - max_face.box.top);if (cur_face_box_area > max_face_box_area){max_face = cur_face;}//檢測輸出處理rockx_image_t out_img;memset(&out_img, 0, sizeof(rockx_image_t));//進行面部矯正對齊ret = rockx_face_align(face_5landmarks_handle, &input_image,&(max_face.box), NULL, &out_img);if (ret != ROCKX_RET_SUCCESS){printf("face_align failed\n");}//人臉特征結果變量定義rockx_face_feature_t out_feature;//獲取人臉特征rockx_face_recognize(face_recognize_handle, &out_img, &out_feature);for (database_iter = database_face_map.begin();database_iter != database_face_map.end(); database_iter++){float similarity;//比較兩個人臉特征的相似性ret = rockx_face_feature_similarity(&database_iter->second,&out_feature, &similarity);printf("simple_value = %lf\n", similarity);//判斷預測精度if (similarity <= 1.0){is_recognize = true;//predict_name_bak = database_iter->first;predict = database_iter->first;break;}else{is_recognize = false;predict = "";continue;}}if (is_recognize == true){predict = database_iter->first;}else{predict = "";}set_rockx_prdict_name(predict);#endif}}
#endif//釋放緩沖區RK_MPI_MB_ReleaseBuffer(src_mb);src_mb = NULL;}//釋放相關人臉處理數據模塊rockx_destroy(face_det_handle);rockx_destroy(face_recognize_handle);rockx_destroy(face_5landmarks_handle);return NULL;
}
人臉識別線程
void *rockx_vi_face_recognize_venc_thread(void *args)
{pthread_detach(pthread_self());//線程資源自動釋放MEDIA_BUFFER mb = NULL; //媒體緩存區int ret;float x_rate = (float)1280 / 1920;float y_rate = (float)720 / 1080;while (!quit){//從指定通道中獲取數據緩沖區mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_RGA, 0, -1);if (!mb){printf("ERROR: RK_MPI_SYS_GetMediaBuffer get null buffer!\n");break;}//獲取人臉處理結果rockx_object_array_t face_array = get_rockx_face_array();//創建mat對象,并創建了720x1080的像素塊,每個像素每個通道的位數都是8位,一個字節的。上述CV_8UC3中的8表示8位、UC表示uchar類型、1表示1個通道Mat tmp_img = Mat(720, 1280, CV_8UC1, RK_MPI_MB_GetPtr(mb));
#if 1//對人臉x,y,w,h進行處理for (int i = 0; i < face_array.count; i++){int x = face_array.object[i].box.left * x_rate;int y = face_array.object[i].box.top * y_rate;int w = (face_array.object[i].box.right - face_array.object[i].box.left) * x_rate;int h = (face_array.object[i].box.bottom - face_array.object[i].box.top) * y_rate;if (x < 0)x = 0;if (y < 0)y = 0;while ((uint32_t)(x + w) >= 1280){w -= 16;}while ((uint32_t)(y + h) >= 720){h -= 16;}//獲取人臉預測名字string predict_name = get_rockx_prdict_name();printf("predict_name = %s\n", predict_name.c_str());nv12_border((char *)RK_MPI_MB_GetPtr(mb), 1280, 720, x, y, w, h, 255, 0, 255);int baseline;//計算人臉名字文本大小Size text_size = getTextSize(predict_name, 2, 2, 2, &baseline);Point origin;origin.x = tmp_img.cols / 4 - text_size.width / 4;origin.y = tmp_img.rows / 4 + text_size.height / 4;//把名字字符填充到文本框里面去cv::putText(tmp_img, predict_name, origin, cv::FONT_HERSHEY_COMPLEX, 1, cv::Scalar(255, 0, 255), 3);}
#endif//釋放對應的資源RK_MPI_SYS_SendMediaBuffer(RK_ID_VENC, 0, mb);RK_MPI_MB_ReleaseBuffer(mb);mb = NULL;}return NULL;
}
人臉編碼rtsp傳輸線程
void *rockx_venc_rtsp_thread(void *args)
{pthread_detach(pthread_self());MEDIA_BUFFER mb = NULL;while (!quit){mb = RK_MPI_SYS_GetMediaBuffer(RK_ID_VENC, 0, -1);if (!mb){printf("ERROR: RK_MPI_SYS_GetMediaBuffer get null buffer!\n");break;}//rtsp來傳輸碼流rtsp_tx_video(g_rtsp_session, (unsigned char *)RK_MPI_MB_GetPtr(mb), RK_MPI_MB_GetSize(mb), RK_MPI_MB_GetTimestamp(mb));RK_MPI_MB_ReleaseBuffer(mb);rtsp_do_event(g_rtsplive);}return NULL;
}