kinect在openni下也能玩抠出人物换背景

之前想了个很拉风的名字《用kinect玩穿越》，但是现在功能还不是很完善，细节处理也不是很好，脸皮没有足够的厚，所以呢还是叫换背景吧。

     这里面包含两个技术要点：

一、抠出活动人物

      在微软的SDK里深度图像的前3位即0-2位就包含有玩家信息，所以提取很方便，可以参考博客http://www.cnblogs.com/yangecnu/archive/2012/04/04/KinectSDK_Depth_Image_Processing_Part1.html

而在openni下深度图的每一位也是两个字节16位，但没有包含这方面的信息，所以就要找其他的方法来实现，在网上找了很久也没有看到有人做了这方面的例子，反过头来还得找openni的用户手册，发现有Scene Analyzer和User Generator可能可以达到我的要求。

   （1）Scene Analyzer：Get Label Map: Provides a map in which each pixel has a meaningful label （它有一个区分前景和背景的功能，但我没有具体去测试）

   （2）User Generator ：Get User Pixels: Provides the pixels that represent the user. The output is a map of the pixels of the entire scene, where the pixels that represent the body are labeled User ID.（它有一个通过用户的ID，来获得用户掩码的功能，我就是用这个函数来做的）

    它的具体函数原型为：GetUserPixels (XnUserID user, SceneMetaData &smd) const

传入的参数为用户的ID和场景图像的引用。

 

二、变换背景

     变换背景的方法为先将要更换的背景图片存入到一个图像数组，然后通过一个索引值index来获取不同的背景图像，那么这个索引值怎么改变呢，这里我用到了一个简单的手势识别，挥右手index加1，挥左手index减1（这里的手势识别是通过骨骼数据的判断来做的具体参考我的上一篇博客），最后将人物图像叠加到背景图片上，就可以实现说是的换背景功能了。说了这么多，先上图吧。

实验结果图：

 图片很模糊是因为投影仪的效果不好。可以看出由于没做边缘融合，图像边缘有很多的锯齿。但做得好的话，设定好人物出现的区域和大小，真的会有穿越的感觉，哈哈！！

代码：

   1 /*********************************************************************************************
  2 //从采集的视频中扣出人物图像,并叠加在背景上
  3 
  4 *******************************************************************************************/
  5 #include <stdlib.h>   
  6 #include <iostream>   
  7 #include <vector>   
  8   
  9 #include <XnCppWrapper.h>   
 10 #include <XnModuleCppInterface.h>    
 11 #include "cv.h"   
 12 #include "highgui.h"   
 13 
 14 
 15 #include <windows.h>
 16 
 17 using namespace std;  
 18 using namespace cv;  
 19   
 20  
 21   
 22 //【1】  三个生成器
 23 xn::UserGenerator userGenerator;  
 24 xn::DepthGenerator depthGenerator;  
 25 xn::ImageGenerator imageGenerator;  
 26   
 27 /* 
 28     XN_SKEL_HEAD          = 1,    XN_SKEL_NECK            = 2, 
 29   XN_SKEL_TORSO         = 3,    XN_SKEL_WAIST           = 4, 
 30     XN_SKEL_LEFT_COLLAR        = 5,    XN_SKEL_LEFT_SHOULDER        = 6, 
 31   XN_SKEL_LEFT_ELBOW        = 7,  XN_SKEL_LEFT_WRIST          = 8, 
 32   XN_SKEL_LEFT_HAND          = 9,    XN_SKEL_LEFT_FINGERTIP    =10, 
 33     XN_SKEL_RIGHT_COLLAR    =11,    XN_SKEL_RIGHT_SHOULDER    =12, 
 34   XN_SKEL_RIGHT_ELBOW        =13,  XN_SKEL_RIGHT_WRIST          =14, 
 35   XN_SKEL_RIGHT_HAND      =15,    XN_SKEL_RIGHT_FINGERTIP    =16, 
 36     XN_SKEL_LEFT_HIP          =17,    XN_SKEL_LEFT_KNEE            =18, 
 37   XN_SKEL_LEFT_ANKLE        =19,  XN_SKEL_LEFT_FOOT            =20, 
 38   XN_SKEL_RIGHT_HIP          =21,    XN_SKEL_RIGHT_KNEE          =22, 
 39     XN_SKEL_RIGHT_ANKLE        =23,    XN_SKEL_RIGHT_FOOT          =24     
 40 */  
 41 //a line will be drawn between start point and corresponding end point   
 42 int startSkelPoints[14]={1,2,6,6,12,17,6,7,12,13,17,18,21,22};  
 43 int endSkelPoints[14]={2,3,12,21,17,21,7,9,13,15,18,20,22,24};  
 44 
 45 //识别函数
 46 void recogGesture(XnPoint3D  skelPointsIn[24], int flag,unsigned int& imgIndex);
 47 
 48   
 49 // callback function of user generator: new user  //回调函数1 new user 
 50 void XN_CALLBACK_TYPE NewUser( xn::UserGenerator& generator, XnUserID user,void* pCookie )  
 51 {  
 52     cout << "New user identified: " << user << endl;  
 53     //userGenerator.GetSkeletonCap().LoadCalibrationDataFromFile( user, "UserCalibration.txt" );   
 54     generator.GetPoseDetectionCap().StartPoseDetection("Psi", user);  
 55 }  
 56   
 57 // callback function of user generator: lost user   //回调函数2 lost user
 58 void XN_CALLBACK_TYPE LostUser( xn::UserGenerator& generator, XnUserID user,void* pCookie )  
 59 {  
 60     cout << "User " << user << " lost" << endl;  
 61 }  
 62   
 63 // callback function of skeleton: calibration start   //回调函数3 calibration start
 64 void XN_CALLBACK_TYPE CalibrationStart( xn::SkeletonCapability& skeleton,XnUserID user,void* pCookie )  
 65 {  
 66     cout << "Calibration start for user " <<  user << endl;  
 67 }  
 68   
 69 // callback function of skeleton: calibration end    //回调函数4 calibraton
 70 void XN_CALLBACK_TYPE CalibrationEnd( xn::SkeletonCapability& skeleton,XnUserID user,XnCalibrationStatus calibrationError,void* pCookie )  
 71 {  
 72     cout << "Calibration complete for user " <<  user << ", ";  
 73     if( calibrationError==XN_CALIBRATION_STATUS_OK )  
 74     {  
 75         cout << "Success" << endl;  
 76         skeleton.StartTracking( user );  
 77         //userGenerator.GetSkeletonCap().SaveCalibrationDataToFile(user, "UserCalibration.txt" );   
 78     }  
 79     else  
 80     {  
 81         cout << "Failure" << endl;  
 82         //For the current version of OpenNI, only Psi pose is available  //重新调用姿势检测函数
 83         ((xn::UserGenerator*)pCookie)->GetPoseDetectionCap().StartPoseDetection( "Psi", user );  
 84     }  
 85 }  
 86   
 87 // callback function of pose detection: pose start   //回调函数5 姿势检测，现只支持Psi姿势
 88 void XN_CALLBACK_TYPE PoseDetected( xn::PoseDetectionCapability& poseDetection,const XnChar* strPose,XnUserID user,void* pCookie)  
 89 {  
 90     cout << "Pose " << strPose << " detected for user " <<  user << endl;  
 91     ((xn::UserGenerator*)pCookie)->GetSkeletonCap().RequestCalibration( user, FALSE );  
 92     poseDetection.StopPoseDetection( user );  
 93 }  
 94   
 95 void clearImg(IplImage* inputimg)  
 96 {  
 97     CvFont font;  
 98     cvInitFont( &font, CV_FONT_VECTOR0,1, 1, 0, 3, 5);  
 99     memset(inputimg->imageData,255,640*480*3);  
100 }  
101   
102   
103 int main( int argc, char** argv )  
104 {  
105     char key=0;  
106   
107    unsigned int imageBGIndex = 0; //背景图片的序号
108     // initial context   
109     xn::Context context;  
110     context.Init();  
111     xn::ImageMetaData imageMD;//openni中的图像数据  
112     
113     //场景人物掩码数据
114     xn::SceneMetaData sceneMD;//人物掩码数据
115 
116     //背景图片数组
117     IplImage* imgSceneBGs[10];
118 
119     char imageBGnames[50]; //背景图片名字
120 
121     for(int i=0;i<10;i++) 
122     {
123         sprintf(imageBGnames,"d:\\pic\\%d.jpg",i);
124         IplImage* imgBackground = cvLoadImage(imageBGnames,1);
125         imgSceneBGs[i] = cvCreateImage(cvSize(640,480),IPL_DEPTH_8U,3);//给这10个指针分配内存
126 
127         cvResize(imgBackground,imgSceneBGs[i],CV_INTER_LINEAR);//改变背景图像大小,存入数组
128     }
129   
130     IplImage* cameraImg=cvCreateImage(cvSize(640,480),IPL_DEPTH_8U,3);  
131 
132     IplImage* imgScene16u=cvCreateImage(cvSize(640,480),IPL_DEPTH_16U,1);//人物掩码原始数据
133 
134     IplImage* imgSceneShow=cvCreateImage(cvSize(640,480),IPL_DEPTH_8U,1);//人物掩码显示数据
135 
136     IplImage* imgSceneRGB = cvCreateImage(cvSize(640,480),IPL_DEPTH_8U,3);//人物RGB显示数据
137 
138     
139 
140     //IplImage* imgSceneBG = cvCreateImage(cvSize(640,480),IPL_DEPTH_8U,3);//背景RGB显示数据
141     
142 
143     IplImage* imgMerge = cvCreateImage(cvSize(640,480),IPL_DEPTH_8U,3);//融合图像
144 
145     cvNamedWindow("311B实验室",0);  
146     //cvNamedWindow("FIGURE",0);
147 
148     // map output mode   
149     XnMapOutputMode mapMode;  
150     mapMode.nXRes = 640;  
151     mapMode.nYRes = 480;  
152     mapMode.nFPS = 30;  
153   
154     // create generator   
155     depthGenerator.Create( context );  
156     depthGenerator.SetMapOutputMode( mapMode );  
157     imageGenerator.Create( context );  
158     userGenerator.Create( context );  
159   
160   //【2】   
161     // Register callback functions of user generator //为userGenerator注册回调函数 
162     XnCallbackHandle userCBHandle;  
163     userGenerator.RegisterUserCallbacks( NewUser, LostUser, NULL, userCBHandle );  
164   
165   //【3】   
166     // Register callback functions of skeleton capability  //为skeletonCap骨架校正注册回调函数 
167     xn::SkeletonCapability skeletonCap = userGenerator.GetSkeletonCap();  
168     skeletonCap.SetSkeletonProfile( XN_SKEL_PROFILE_ALL );  
169     XnCallbackHandle calibCBHandle;  
170     skeletonCap.RegisterToCalibrationStart( CalibrationStart,&userGenerator, calibCBHandle );  
171     skeletonCap.RegisterToCalibrationComplete( CalibrationEnd,&userGenerator, calibCBHandle );  
172   
173   //【4】   
174     // Register callback functions of Pose Detection capability   
175     XnCallbackHandle poseCBHandle;  
176     userGenerator.GetPoseDetectionCap().RegisterToPoseDetected( PoseDetected,&userGenerator, poseCBHandle );  
177   
178 
179     //【4-1】矫正视角
180      depthGenerator.GetAlternativeViewPointCap().SetViewPoint( imageGenerator );
181     // start generate data   
182     context.StartGeneratingAll();  
183 
184     
185 
186     while( key!=27 )  
187     {  
188         context.WaitAndUpdateAll();  
189   
190         imageGenerator.GetMetaData(imageMD);  
191         memcpy(cameraImg->imageData,imageMD.Data(),640*480*3);  
192         cvCvtColor(cameraImg,cameraImg,CV_RGB2BGR);  
193         // get users   
194         XnUInt16 userCounts = userGenerator.GetNumberOfUsers();  
195         if( userCounts > 0 )  
196         {  
197             XnUserID* userID = new XnUserID[userCounts];  
198             userGenerator.GetUsers( userID, userCounts );  
199 
200             
201             for( int i = 0; i < userCounts; ++i ) //循环每个用户 
202             {  
203 
204 
205                 //获取用户掩码的图像
206                 userGenerator.GetUserPixels(userID[i],sceneMD);
207                 memcpy(imgScene16u->imageData,sceneMD.Data(),640*480*2);//复制内存
208                 cvConvertScale(imgScene16u,imgSceneShow,255/4.0,0);
209 
210 
211 
212                 //【5】   
213                 // if is tracking skeleton   
214                 if( skeletonCap.IsTracking( userID[i] ) )  
215                 {  
216                     XnPoint3D skelPointsIn[24],skelPointsOut[24];  
217                     XnSkeletonJointTransformation mJointTran;  
218                     for(int iter=0;iter<24;iter++)  
219                     {  
220                         //XnSkeletonJoint from 1 to 24             
221                                                 skeletonCap.GetSkeletonJoint( userID[i],XnSkeletonJoint(iter+1), mJointTran );  
222                         skelPointsIn[iter]=mJointTran.position.position;  
223 
224                         
225                     }  
226 
227                     //识别动作并发送按键消息
228                      recogGesture(skelPointsIn, i%2,imageBGIndex);
229 
230                     //将数据转换到投影坐标系
231                     depthGenerator.ConvertRealWorldToProjective(24,skelPointsIn,skelPointsOut);  
232 
233                     //【6】画图   
234                 /*    for(int d=0;d<1;d++)  
235                     {  
236                         CvPoint startpoint = cvPoint(skelPointsOut[startSkelPoints[d]-1].X,skelPointsOut[startSkelPoints[d]-1].Y);  
237                         CvPoint endpoint = cvPoint(skelPointsOut[endSkelPoints[d]-1].X,skelPointsOut[endSkelPoints[d]-1].Y);  
238               
239                         cvCircle(cameraImg,startpoint,5,CV_RGB(0,0,255),12);  
240                        // cvCircle(cameraImg,endpoint,3,CV_RGB(0,0,255),12);  
241                        // cvLine(cameraImg,startpoint,endpoint,CV_RGB(0,0,255),4);  
242                     }  */
243                 }  
244             }  
245             delete [] userID;  
246         }  
247         //memset(imgSceneRGB->imageData,0,640*480*3); //显示数据清0
248         
249         //cvCopy(cameraImg,imgSceneRGB,imgSceneShow);
250 
251         cvCopy(imgSceneBGs[imageBGIndex%10],imgMerge); //选择一个背景并复制背景图像
252 
253         //cvAdd(imgMerge, cameraImg,imgMerge,imgSceneShow);//加入抠出的人物图像
254         
255         cvCopy(cameraImg,imgMerge,imgSceneShow);//加入抠出的人物图像
256 
257         cvShowImage("311B实验室",imgMerge);  
258         //cvShowImage("FIGURE",imgSceneShow);
259   
260         key=cvWaitKey(30);  
261         
262   
263     }  
264     // stop and shutdown   
265     cvDestroyWindow("Camera");  
266     cvReleaseImage(&cameraImg);  
267     context.StopGeneratingAll();  
268     context.Shutdown();  
269   
270     return 0;  
271 }  
272 
273 /*********************************************************************************************
274 //动作识别函数，
275 （根据骨架坐标点识别动作，并触发相应的虚拟按键，后期这两个功能需要用两个函数实现）
276 //输入：skelPointsIn[24]骨骼数据，用户标识 flag，背景图片序号
277 
278 **********************************************************************************************/
279 void recogGesture(XnPoint3D  skelPointsIn[24], int flag,unsigned int& imgIndex)
280 {
281                     
282         //上一帧手的z坐标
283        static  float preLeftHandZ, preRightHandZ;
284     
285         //获取头和右手的坐标点
286         XnPoint3D skelPointHead,skelPointRightHand,skelPointLeftHand;
287         //躯干中心
288         XnPoint3D skelPointTorso;
289         //XnPoint3D skelPointRightTip;
290         skelPointHead = skelPointsIn[XN_SKEL_HEAD-1];
291         skelPointRightHand = skelPointsIn[XN_SKEL_RIGHT_HAND-1];
292         skelPointLeftHand = skelPointsIn[XN_SKEL_LEFT_HAND-1];
293 
294         //躯干中心点数据
295         skelPointTorso = skelPointsIn[XN_SKEL_TORSO-1];
296          
297          
298         //判断右手出拳动作
299         if(skelPointHead.Z-skelPointRightHand.Z>400.0&&// 大于头部坐标一定位置
300             skelPointHead.Z-preRightHandZ<=400.0&&
301             skelPointRightHand.Y-skelPointTorso.Y>50)//高于躯干中心100
302             {
303                 /*//发送按键->
304                 keybd_event(37,0,0,0);
305                 Sleep(15); //不知道有没有用？？
306                 keybd_event(37,0,KEYEVENTF_KEYUP,0);*/
307                 ++imgIndex ;
308             
309                         
310             }
311 
312         //判断左手出拳动作
313         if(skelPointHead.Z -skelPointLeftHand.Z >400.0&&//大于头部Z坐标480
314             skelPointHead.Z -preLeftHandZ<=400.0&&
315             skelPointLeftHand.Y-skelPointTorso.Y>50)//高于躯干中心100
316         {
317 
318 
319             /*    //发送按键<-
320             keybd_event(39,0,0,0);
321             Sleep(15);
322             keybd_event(39,0,KEYEVENTF_KEYUP,0);*/
323 
324             --imgIndex;
325 
326                         
327         }
328 
329         //判断前进动作，两手低下向前
330     /*if(skelPointHead.Z-skelPointRightHand.Z>150&&skelPointHead.Z-skelPointLeftHand.Z>150&&//双手手Z<头Z200
331             skelPointTorso.Y-skelPointRightHand.Y>150&&skelPointTorso.Y-skelPointLeftHand.Y>150) //双手Y低于躯干Y200
332             
333         {
334     
335             //发送按键F5
336             keybd_event(116,0,0,0);
337             Sleep(15);
338             keybd_event(116,0,KEYEVENTF_KEYUP,0);
339         
340         }*/
341 
342         //判断后退动作，两手低下向后
343 
344     /*        if(skelPointRightHand.Z-skelPointHead.Z>150&&skelPointLeftHand.Z-skelPointHead.Z>150&& //双手手Z>头Z200
345             skelPointTorso.Y-skelPointRightHand.Y>150&&skelPointTorso.Y-skelPointLeftHand.Y>150)//双手Y低于躯干Y200
346         {
347             //发送按键A
348             keybd_event('A',0,0,0);
349             Sleep(5);
350             keybd_event('A',0,KEYEVENTF_KEYUP,0);
351         
352         
353         */
354             
355         //保留手坐标的历史数据
356         preLeftHandZ = skelPointLeftHand.Z;
357         preRightHandZ = skelPointRightHand.Z;
358     
359 
360 }

代码没时间来整理优化，写的比较粗糙，欢迎大家拍砖。其中骨骼数据提取参考了小斤的博客http://blog.csdn.net/chenxin_130/article/details/6950480。