亚博microros小车-原生ubuntu支持系列：7-脸部检测

背景知识

官网介绍：

Face Mesh - mediapipe

mpFaceMesh.FaceMesh() 类的参数有：self.staticMode, self.maxFaces, self.minDetectionCon, self.minTrackCon

    staticMode:是否将每帧图像作为静态图像处理。如果为 True，每帧都会进行人脸检测；如果为 False，在检测到人脸后进行跟踪，速度更快
    maxFaces:要检测的最大人脸数量
    minDetectionCon:检测的最小置信度阈值。低于此值的人脸将被忽略
    minTrackCon:跟踪的最小置信度阈值。低于此值的跟踪将被忽略

import cv2
import mediapipe as mpmp_drawing = mp.solutions.drawing_utils#绘图工具
mp_facemesh = mp.solutions.face_mesh
#手部模型
faceMesh = mp_facemesh.FaceMesh(static_image_mode=False,max_num_faces=2,min_detection_confidence=0.75,min_tracking_confidence=0.75)cap = cv2.VideoCapture(0)#打开默认摄像头
while True:ret,frame = cap.read()#读取一帧图像#图像格式转换frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)# 因为摄像头是镜像的，所以将摄像头水平翻转# 不是镜像的可以不翻转frame= cv2.flip(frame,1)#输出结果results = faceMesh.process(frame)frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)if results.multi_face_landmarks:for face_landmarks in results.multi_face_landmarks:# 关键点可视化mp_drawing.draw_landmarks(frame, face_landmarks, mp_facemesh.FACEMESH_CONTOURS)cv2.imshow('MediaPipe face', frame)if cv2.waitKey(1) & 0xFF == 27:break
cap.release()

这几个基本格式类似的，就是换个模型，输出结果不同

效果：

脸部检测

src/yahboom_esp32_mediapipe/yahboom_esp32_mediapipe/目录下新建文件04_FaceMesh.py

代码如下

#!/usr/bin/env python3
# encoding: utf-8
#import ros lib
import rclpy
from rclpy.node import Node
from geometry_msgs.msg import Point
import mediapipe as mp
#import define msg
from yahboomcar_msgs.msg import PointArray
#import commom lib
import cv2 as cv
import numpy as np
import time
from cv_bridge import CvBridge
from sensor_msgs.msg import Image, CompressedImagefrom rclpy.time import Time
import datetimeprint("import done")class FaceMesh(Node):def __init__(self, name,staticMode=False, maxFaces=2, minDetectionCon=0.5, minTrackingCon=0.5):super().__init__(name)self.mpDraw = mp.solutions.drawing_utils#画图self.mpFaceMesh = mp.solutions.face_mesh#模型初始化self.faceMesh = self.mpFaceMesh.FaceMesh(static_image_mode=staticMode,max_num_faces=maxFaces,min_detection_confidence=minDetectionCon,min_tracking_confidence=minTrackingCon )self.pub_point = self.create_publisher(PointArray,'/mediapipe/points',1000)self.lmDrawSpec = mp.solutions.drawing_utils.DrawingSpec(color=(0, 0, 255), thickness=-1, circle_radius=3)self.drawSpec = self.mpDraw.DrawingSpec(color=(0, 255, 0), thickness=1, circle_radius=1)def pubFaceMeshPoint(self, frame, draw=True):pointArray = PointArray()img = np.zeros(frame.shape, np.uint8)imgRGB = cv.cvtColor(frame, cv.COLOR_BGR2RGB)self.results = self.faceMesh.process(imgRGB)#检测if self.results.multi_face_landmarks:for i in range(len(self.results.multi_face_landmarks)):#输出关键点if draw: self.mpDraw.draw_landmarks(frame, self.results.multi_face_landmarks[i], self.mpFaceMesh.FACEMESH_CONTOURS, self.lmDrawSpec, self.drawSpec)self.mpDraw.draw_landmarks(img, self.results.multi_face_landmarks[i], self.mpFaceMesh.FACEMESH_CONTOURS, self.lmDrawSpec, self.drawSpec)for id, lm in enumerate(self.results.multi_face_landmarks[i].landmark):point = Point()point.x, point.y, point.z = lm.x, lm.y, lm.zpointArray.points.append(point)self.pub_point.publish(pointArray)return frame, imgdef frame_combine(slef,frame, src):if len(frame.shape) == 3:frameH, frameW = frame.shape[:2]srcH, srcW = src.shape[:2]dst = np.zeros((max(frameH, srcH), frameW + srcW, 3), np.uint8)dst[:, :frameW] = frame[:, :]dst[:, frameW:] = src[:, :]else:src = cv.cvtColor(src, cv.COLOR_BGR2GRAY)frameH, frameW = frame.shape[:2]imgH, imgW = src.shape[:2]dst = np.zeros((frameH, frameW + imgW), np.uint8)dst[:, :frameW] = frame[:, :]dst[:, frameW:] = src[:, :]return dstclass MY_Picture(Node):def __init__(self, name):super().__init__(name)self.bridge = CvBridge()self.sub_img = self.create_subscription(CompressedImage, '/espRos/esp32camera', self.handleTopic, 1) #获取esp32传来的图像self.last_stamp = Noneself.new_seconds = 0self.fps_seconds = 1self.face_mesh = FaceMesh('face_mesh')def handleTopic(self, msg):self.last_stamp = msg.header.stamp  if self.last_stamp:total_secs = Time(nanoseconds=self.last_stamp.nanosec, seconds=self.last_stamp.sec).nanosecondsdelta = datetime.timedelta(seconds=total_secs * 1e-9)seconds = delta.total_seconds()*100if self.new_seconds != 0:self.fps_seconds = seconds - self.new_secondsself.new_seconds = seconds#保留这次的值start = time.time()frame = self.bridge.compressed_imgmsg_to_cv2(msg)frame = cv.resize(frame, (640, 480))cv.waitKey(10)frame, img = self.face_mesh.pubFaceMeshPoint(frame,draw=False)end = time.time()fps = 1 / ((end - start)+self.fps_seconds)text = "FPS : " + str(int(fps))cv.putText(frame, text, (20, 30), cv.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 1)dist = self.face_mesh.frame_combine(frame, img)cv.imshow('dist', dist)# print(frame)cv.waitKey(10)def main():print("start it")rclpy.init()esp_img = MY_Picture("My_Picture")try:rclpy.spin(esp_img)except KeyboardInterrupt:passfinally:esp_img.destroy_node()rclpy.shutdown()

订阅esp32传出来的图像后，通过MediaPipe去做相关的识别后显示。主体流程跟之前一样，换了检测模型。

构建后运行：ros2 run yahboom_esp32_mediapipe FaceMesh

效果如下