人体姿态估计库 mediapipe

在jetson-nx上文件夹中的whl包就能安装了，在PC的windows上直接pip install mediapipe就能安装

whl包参考零基础入门Jetson Nano——MediaPipe双版本（CPU+GPU）的安装与使用_mediapipe gpu-CSDN博客

1 全身姿态检测

1.1 基本使用

1.2 关键点分析

1.2.1 确定对应关系

1.2.2 通过对应关系判断

1.3 搭配摄像头使用

2 手部姿态检测

2.1 基本使用

2.2 关键点分析

2.2.1 确定对应关系

2.2.2 通过对应关系判断

2.3 搭配摄像头使用

1 全身姿态检测

1.1 基本使用

import cv2
import mediapipe as mp
from PIL import ImageFont,ImageDraw,Imagemp_drawing = mp.solutions.drawing_utils
mp_pose = mp.solutions.pose
pose = mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5)image = cv2.imread('right_foot_aboard.jpg')
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)# To improve performance, optionally mark the image as not writeable to
# pass by reference.
image.flags.writeable = False
results = pose.process(image)# Draw the pose annotation on the image.
image.flags.writeable = True
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)pose.close()
cv2.imshow('MediaPipe Pose', image)
cv2.waitKey(0)
cv2.destroyAllWindows()

1.2 关键点分析

1.2.1 确定对应关系

我们可以对识别到的点进行操作，比如检测左侧脚尖在上，还是右脚脚尖在上，也就是32点和31点哪一个在y轴值更小

首先打印一下result

一共是33个点，我们看最后的31和32

我们可以根据图与结果分析出来下面几个信息

x轴和y轴以图像的左上角为原点，值越大越靠右或靠下。z轴以屏幕为原点，值越小距离屏幕越远
x,y,z是比例值。x估计是与图像宽的比例，y估计是与图像高的比例，z不知道
visibility是可见度，值越大越可见，值越小说明可能被遮挡

1.2.2 通过对应关系判断

我们可以通过遍历拿到左脚、右脚脚尖与图像的比例，然后把他们进行比较

我们换一张图测试一下

1.3 搭配摄像头使用

import cv2
import mediapipe as mp
from PIL import ImageFont, ImageDraw, Imagemp_drawing = mp.solutions.drawing_utils
mp_pose = mp.solutions.pose
pose = mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5)cap = cv2.VideoCapture(0)
while True:ret, frame = cap.read()if ret:image = cv2.cvtColor(cv2.flip(frame, 1), cv2.COLOR_BGR2RGB)image.flags.writeable = Falseresults = pose.process(image)image.flags.writeable = Trueimage = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)cv2.imshow("capture", image)k = cv2.waitKey(1)if k == ord(' ') :break
# 关闭视频捕获器
cap.release()
# 销毁所有窗口
cv2.destroyAllWindows()
pose.close()

测试过在jetson-nx上用GPU是流畅的。在PC上CPU(CPU与内存配置如下图)是流畅的(GPU没测)

2 手部姿态检测

参考 JetBot手势识别实验_from jetbot import robot-CSDN博客

2.1 基本使用

import cv2
import mediapipe as mp# 初始化MediaPipe Hands模块
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(static_image_mode=True,max_num_hands=2,min_detection_confidence=0.5,min_tracking_confidence=0.5)mp_drawing = mp.solutions.drawing_utils  # 用于绘制关键点的工具# 读取图片
image_path = '1.jpg'  # 这里替换为你的图片路径
image = cv2.imread(image_path)if image is None:print("Cannot find the image.")
else:# 将图像从BGR转换为RGBimage_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)# 处理图像，检测手部results = hands.process(image_rgb)# 将图像从RGB转回BGR以显示image_bgr = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR)# 绘制手部关键点if results.multi_hand_landmarks:for hand_landmarks in results.multi_hand_landmarks:mp_drawing.draw_landmarks(image_bgr, hand_landmarks, mp_hands.HAND_CONNECTIONS)# 显示图像cv2.imshow('Hand Detection', image_bgr)cv2.waitKey(0)  # 等待按键cv2.destroyAllWindows()# # 可选：保存输出图像# output_image_path = 'path_to_your_output_image.jpg'  # 输出文件的路径# cv2.imwrite(output_image_path, image_bgr)# print("Output image is saved as", output_image_path)# 释放资源
hands.close()

2.2 关键点分析

2.2.1 确定对应关系

与全身姿态用法相同了，我们简单说一下

results.multi_hand_landmarks[0]可能会有直接变成列表的方法，我这里就直接用正则取了

import cv2
import mediapipe as mp
import rep = re.compile(r'landmark {\n  x: .*\n  y: .*\n  z: .*\n}')
# 初始化MediaPipe Hands模块
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(static_image_mode=True,max_num_hands=2,min_detection_confidence=0.5,min_tracking_confidence=0.5)mp_drawing = mp.solutions.drawing_utils  # 用于绘制关键点的工具# 读取图片
image_path = '1.jpg'  # 这里替换为你的图片路径
image = cv2.imread(image_path)if image is None:print("Cannot find the image.")
else:# 将图像从BGR转换为RGBimage_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)# 处理图像，检测手部results = hands.process(image_rgb)# print()id = 0for result in p.findall(str(results.multi_hand_landmarks[0])):print('id',id)id = id + 1print(result)print()# 将图像从RGB转回BGR以显示image_bgr = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR)# 绘制手部关键点if results.multi_hand_landmarks:for hand_landmarks in results.multi_hand_landmarks:mp_drawing.draw_landmarks(image_bgr, hand_landmarks, mp_hands.HAND_CONNECTIONS)cv2.imshow('Hand Detection', image_bgr)cv2.waitKey(0)  # 等待按键cv2.destroyAllWindows()hands.close()

我们重点关注食指，通过点位图来看不是8就16

对比图像来看，食指的顶点应该过图像的一半，所以16不符合。之后考虑8，观察其他点与8点的y值可以看出8点比其他的点的y值要小，基本可以断定8点就是食指的顶点

2.2.2 通过对应关系判断

我们比如要判断食指是否伸展，那么就判断 |8的y-0的y| 是否大于 |7的y-0的y|

上面这个条件仅考虑手掌面对摄像头或背对摄像头的情况，不考虑手掌冲上或冲下的情况

import cv2
import mediapipe as mp
import rep = re.compile(r'y: (.*)')
# 初始化MediaPipe Hands模块
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(static_image_mode=True,max_num_hands=2,min_detection_confidence=0.5,min_tracking_confidence=0.5)mp_drawing = mp.solutions.drawing_utils  # 用于绘制关键点的工具# 读取图片
image_path = '6.jpg'  # 这里替换为你的图片路径
image = cv2.imread(image_path)if image is None:print("Cannot find the image.")
else:# 将图像从BGR转换为RGBimage_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)# 处理图像，检测手部results = hands.process(image_rgb)y_list = p.findall(str(results.multi_hand_landmarks[0]))if abs(float(y_list[8])-float(y_list[0]))>abs(float(y_list[7])-float(y_list[0])):print('食指伸展')else:print('食指不伸展')# 将图像从RGB转回BGR以显示image_bgr = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR)# 绘制手部关键点if results.multi_hand_landmarks:for hand_landmarks in results.multi_hand_landmarks:mp_drawing.draw_landmarks(image_bgr, hand_landmarks, mp_hands.HAND_CONNECTIONS)cv2.imshow('Hand Detection', image_bgr)cv2.waitKey(0)  # 等待按键cv2.destroyAllWindows()hands.close()

换一张图测一下

如果要通过5个手指的判断手势是有点复杂的，而且mediapipe也不一定准，所以不建议通过mediapipe做手势识别

2.3 搭配摄像头使用

注意在代码中把static_image_mode改为False

import cv2
import mediapipe as mpmp_hands = mp.solutions.hands
hands = mp_hands.Hands(static_image_mode=False,max_num_hands=2,min_detection_confidence=0.5,min_tracking_confidence=0.5)mp_drawing = mp.solutions.drawing_utilscap = cv2.VideoCapture(0)
while True:ret, frame = cap.read()if ret:image_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)results = hands.process(image_rgb)image_bgr = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR)if results.multi_hand_landmarks:for hand_landmarks in results.multi_hand_landmarks:mp_drawing.draw_landmarks(image_bgr, hand_landmarks, mp_hands.HAND_CONNECTIONS)cv2.imshow("capture", image_bgr)k = cv2.waitKey(1)if k == ord(' '):breakcap.release()
cv2.destroyAllWindows()

测试环境与全身姿态检测相同，运行流畅。在nx中会出现下面两个warning，就结果来看问题不大