Hardware:
Jetson Nano
Camera: Some cheap USB Webcam by foscam
Servo Driver: PCA9685 with 5-6V Power Supply
Servos: S-7361
Setup:
Etch this image (e.g. with Balena Etcher)
https://developer.nvidia.com/jetson-nano-sd-card-imageHousekeeping:
sudo apt update
sudo apt autoremove -y
sudo apt clean
sudo apt remove thunderbird libreoffice-* -y
sudo apt-get install -y python3-pip
sudo pip3 install -U pip testresources setuptoolsInstall OpenCV
sudo apt-get install python3-opencv
sudo apt-get remove python3-opencvYou can import cv2 and check the version with
cv2.__version__You should end up with at least version 4.1.1 (depending on updates)
Install Mediapipe
https://github.com/anion0278/mediapipe-jetson#installation-for-clean-jetpack-46—461—python-wheel
Get binaries there, download into a folder like /home/mediapipe
Go there in terminal and run:
### Preparing pip
sudo apt update
sudo apt install python3-pip
pip3 install --upgrade pip
### Remove previous versions of Mediapipe (if it was installed):
pip3 uninstall mediapipe
### Install from wheel with (run commands from mediapipe dir):
pip3 install protobuf==3.19.4 opencv-python==4.5.3.56 dataclasses mediapipe-0.8.9_cuda102-cp36-cp36m-linux_aarch64.whl
### Note: Building wheel for newer version of opencv-python may take quite some time (up to a few hours)!Get Circuitpython Servokit
parts from: https://learn.adafruit.com/circuitpython-libraries-on-linux-and-the-nvidia-jetson-nano/initial-setup
Terminal:
sudo pip3 install -U \
adafruit-circuitpython-busdevice==5.1.2 \
adafruit-circuitpython-motor==3.3.5 \
adafruit-circuitpython-pca9685==3.4.1 \
adafruit-circuitpython-register==1.9.8 \
adafruit-circuitpython-servokit==1.3.8 \
Adafruit-Blinka==6.11.1 \
Adafruit-GPIO==1.0.3 \
Adafruit-MotorHAT==1.4.0 \
Adafruit-PlatformDetect==3.19.6 \
Adafruit-PureIO==1.1.9 \
Adafruit-SSD1306==1.6.2
pip3 freeze - local | grep -v '^\-e' | cut -d = -f 1 | xargs -n1 pip3 install -U
sudo bash
pip3 freeze - local | grep -v '^\-e' | cut -d = -f 1 | xargs -n1 pip3 install -U
sudo /opt/nvidia/jetson-io/jetson-io.pySelect Configure 40-pin expansion header at the bottom. Then select Configure header pins manually.
Select spi1 (19, 21, 23, 24, 26) and then select Back
Finally select Save pin changes and then Save and reboot to reconfigure pins. This will create a config file and reboot the Jetson Nano.
adjust permissions:
sudo groupadd -f -r gpio
# use your username instead of "benno". If you are unsure, what it is, just run "whoami"
sudo usermod -a -G gpio benno
sudo usermod -a -G i2c benno
# possibly: sudo usermod -aG sudo username
# to check:
getent group | grep benno
cd ~
git clone https://github.com/NVIDIA/jetson-gpio.git
sudo cp ~/jetson-gpio/lib/python/Jetson/GPIO/99-gpio.rules /etc/udev/rules.d
sudo chown root.gpio /dev/gpiochip0
sudo chmod 660 /dev/gpiochip0
sudo udevadm control --reload-rules && sudo udevadm trigger
reboot
Check if you device gets found with
sudo i2cdetect -r -y 1Download Visual Studio
Download Visual Studio arm64 .deb from:
https://code.visualstudio.com/download
Install by just double clicking.
Launch Visual Studio Code
Select Extensions
install extension „Python“
Ctrl-Shift+P > Select interpreter > Python 3.6
Create New File with .py ending. e.g. OpenCVServos.py
import cv2
import sys, time
from adafruit_servokit import ServoKit
import mediapipe as mp
import math
kit = ServoKit(channels=16)
servo0 = 90
servo1 = 90
servo2 = 90
servo3 = 90
servo4 = 90
servo5 = 90
kit.servo[0].angle = servo0
kit.servo[1].angle = servo1
kit.servo[2].angle = servo2
kit.servo[3].angle = servo3
kit.servo[4].angle = servo4
kit.servo[5].angle = servo5
mp_drawing = mp.solutions.drawing_utils
mp_face_mesh = mp.solutions.face_mesh
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
face_mesh = mp_face_mesh.FaceMesh(static_image_mode=False,
max_num_faces=1,
min_detection_confidence=0.8)
def get_face_landmarks(image):
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
if not results.multi_face_landmarks:
return []
return results.multi_face_landmarks[0].landmark # Assuming only one face
# Calculate Euclidean distance between two landmarks
def calculate_distance(landmark1, landmark2):
x1, y1, z1 = landmark1.x, landmark1.y, landmark1.z
x2, y2, z2 = landmark2.x, landmark2.y, landmark2.z
distance = math.sqrt((x1 - x2)**2 + (y1 - y2)**2 + (z1 - z2)**2)
return distance
# Map a value from one range to another
def map_value(value, from_min, from_max, to_min, to_max):
return (value - from_min) / (from_max - from_min) * (to_max - to_min) + to_min
def get_face_mesh(image):
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
# Print and draw face mesh landmarks on the image.
if not results.multi_face_landmarks:
return image
annotated_image = image.copy()
for face_landmarks in results.multi_face_landmarks:
# uncomment for live printout of all coordiates
# print(' face_landmarks:', face_landmarks)
mp_drawing.draw_landmarks(
image=annotated_image,
landmark_list=face_landmarks,
connections=mp_face_mesh.FACEMESH_CONTOURS,
# Change to FACEMESH_TESSELATION for mesh view
landmark_drawing_spec=drawing_spec,
connection_drawing_spec=drawing_spec)
# uncomment for printout of total number of processed landmarks
# print('%d facemesh_landmarks' % len(face_landmarks.landmark))
if len(face_landmarks.landmark) >= 2:
# Setup compared landmark cuples
# right eye
landmark1 = face_landmarks.landmark[23]
landmark2 = face_landmarks.landmark[223]
# left eye
landmark3 = face_landmarks.landmark[253]
landmark4 = face_landmarks.landmark[442]
# right Mundwinkel
landmark5 = face_landmarks.landmark[57]
landmark6 = face_landmarks.landmark[240]
# left Mundwinkel
landmark7 = face_landmarks.landmark[287]
landmark8 = face_landmarks.landmark[260]
# Mundbreite
landmark9 = face_landmarks.landmark[57]
landmark10 = face_landmarks.landmark[287]
# Augenlider
landmark11 = face_landmarks.landmark[257]
landmark12 = face_landmarks.landmark[374]
# Calculate the distance between the two landmarks
distance0 = calculate_distance(landmark1, landmark2)
distance1 = calculate_distance(landmark3, landmark4)
distance2 = calculate_distance(landmark5, landmark6)
distance3 = calculate_distance(landmark7, landmark8)
distance4 = calculate_distance(landmark9, landmark10)
distance5 = calculate_distance(landmark11, landmark12)
# print(distance0)
if distance0<0.03:
distance0=0.03
if distance0>0.08:
distance0=0.08
# Map the distance to servo angle (adjust the mapping values as needed)
servo_angle0 = int(map_value(distance0, 0.03, 0.08, 0, 180))
# print(servo_angle0)
# Set servo angle
kit.servo[0].angle = servo_angle0
# print(distance1)
if distance1<0.03:
distance1=0.03
if distance1>0.08:
distance1=0.08
servo_angle1 = int(map_value(distance1, 0.03, 0.08, 0, 180))
# print(servo_angle1)
kit.servo[1].angle = servo_angle1
# print(distance2)
if distance2<0.03:
distance2=0.03
if distance2>0.08:
distance2=0.08
servo_angle2 = int(map_value(distance2, 0.03, 0.08, 0, 180))
# print(servo_angle2)
kit.servo[2].angle = servo_angle2
# print(distance3)
if distance3<0.03:
distance3=0.03
if distance3>0.08:
distance3=0.08
servo_angle3 = int(map_value(distance3, 0.03, 0.08, 0, 180))
# print(servo_angle3)
kit.servo[3].angle = servo_angle3
# print(distance4)
if distance4<0.03:
distance4=0.03
if distance4>0.08:
distance4=0.08
servo_angle4 = int(map_value(distance4, 0.03, 0.08, 0, 180))
# print(servo_angle4)
kit.servo[4].angle = servo_angle4
# print(distance5)
if distance5<0.03:
distance5=0.03
if distance5>0.08:
distance5=0.08
servo_angle5 = int(map_value(distance5, 0.03, 0.08, 0, 180))
# print(servo_angle5)
kit.servo[5].angle = servo_angle5
return annotated_image
font = cv2.FONT_HERSHEY_SIMPLEX
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
if (cap.isOpened() == False):
print("Unable to read camera feed")
while cap.isOpened():
s = time.time()
ret, img = cap.read()
if ret == False:
print('WebCAM Read Error')
sys.exit(0)
annotated = get_face_mesh(img)
#you can do the same for the other landmarks and servos.
e = time.time()
fps = 1 / (e - s)
cv2.putText(annotated, 'FPS:%5.2f' % (fps), (10, 50), font, fontScale=1, color=(0, 255, 0), thickness=1)
cv2.imshow('webcam', annotated)
key = cv2.waitKey(1)
if key == 27: # ESC
break
cap.release()
Mediapipe References:
Overview:
https://github.com/google/mediapipe/blob/master/docs/solutions/face_mesh.md
Face Landmark name list
Image with landmark numbers overlayed:
Explanation of extraction mode for individual landmark coordinates:
Camera:
This implementation of mediapipe does not support gstreamer. That means: only USB-Cameras!
Here is what I used to get mine to work:
https://github.com/jetsonhacks/USB-Camera/blob/main/usb-camera-simple.py
https://stackoverflow.com/questions/64272731/open-cv-shows-green-screen-on-jetson-nano
pip install opencv-contrib-python
further resources:
Starting the Raspberry Pi Camera, or a WEB Camera on the Jetson Nano
Great tutorials! Thank you very much. These gave me all the basics I needed.