Skip to content

MediaPipe FaceMesh

Detect the lower face with MediaPipe's FaceMesh detector.

This detector is based on the face landmarker task from MediaPipe. The face landmarker provides locations of more than 450 facial landmarks. From these, we can define a region for the lower face, as is done for example by Li et al. (2014)1.

  1. X. Li, J. Chen, G. Zhao, and M. Pietikainen, “Remote Heart Rate Measurement From Face Videos Under Realistic Situations”, Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR), pp. 4264-4271, 2014 doi:10.1109/CVPR.2014.543 ↩

FaceMeshDetector

Bases: RoiDetector

Face detector using MediaPipe's face landmarker.

Source code in src\yarppg\roi\facemesh_segmenter.py 
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
class FaceMeshDetector(RoiDetector):
    """Face detector using MediaPipe's face landmarker."""

    _lower_face = [200, 431, 411, 340, 349, 120, 111, 187, 211]

    def __init__(self, draw_landmarks=False, **kwargs):
        super().__init__(**kwargs)
        modelpath = get_face_landmarker_modelfile()
        if modelpath is None:
            raise FileNotFoundError("Could not find or download landmarker model file.")
        base_options = mp.tasks.BaseOptions(model_asset_path=modelpath)
        landmarker_options = mp.tasks.vision.FaceLandmarkerOptions(
            base_options=base_options,
            running_mode=mp.tasks.vision.RunningMode.VIDEO,
        )
        self.landmarker = mp.tasks.vision.FaceLandmarker.create_from_options(
            landmarker_options
        )
        self.draw_landmarks = draw_landmarks

    def __del__(self):
        self.landmarker.close()

    def _process_landmarks(self, frame, results) -> tuple[np.ndarray, np.ndarray]:
        height, width = frame.shape[:2]
        coords = get_landmark_coords(results.face_landmarks[0], width, height)[:, :2]
        face_rect = get_boundingbox_from_coords(coords)

        mask = contour_to_mask((height, width), coords[self._lower_face])
        return mask, face_rect

    def detect(self, frame: np.ndarray) -> RegionOfInterest:
        """Find face landmarks and create ROI around the lower face region."""
        rawimg = frame.copy()
        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            results = self.landmarker.detect_for_video(
                mp_image, int(time.perf_counter() * 1000)
            )

        if len(results.face_landmarks) < 1:
            return RegionOfInterest(np.zeros_like(frame), baseimg=frame)

        if self.draw_landmarks:
            self.draw_facemesh(frame, results.face_landmarks[0], tesselate=True)

        mask, face_rect = self._process_landmarks(frame, results)
        return RegionOfInterest(mask, baseimg=rawimg, face_rect=tuple(face_rect))

    def draw_facemesh(
        self,
        img,
        face_landmarks,
        tesselate=False,
        contour=False,
        irises=False,
    ):
        """Draw the detected face landmarks on the image."""
        face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()  # type: ignore
        face_landmarks_proto.landmark.extend(
            [
                landmark_pb2.NormalizedLandmark(  # type: ignore
                    x=landmark.x, y=landmark.y, z=landmark.z
                )
                for landmark in face_landmarks
            ]
        )
        if tesselate:
            mp.solutions.drawing_utils.draw_landmarks(  # type: ignore
                image=img,
                landmark_list=face_landmarks_proto,
                connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,  # type: ignore
                landmark_drawing_spec=None,
                connection_drawing_spec=TESSELATION_SPEC,
            )
        if contour:
            mp.solutions.drawing_utils.draw_landmarks(  # type: ignore
                image=img,
                landmark_list=face_landmarks_proto,
                connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,  # type: ignore
                landmark_drawing_spec=None,
                connection_drawing_spec=CONTOUR_SPEC,
            )
        if irises:
            mp.solutions.drawing_utils.draw_landmarks(  # type: ignore
                image=img,
                landmark_list=face_landmarks_proto,
                connections=mp.solutions.face_mesh.FACEMESH_IRISES,  # type: ignore
                landmark_drawing_spec=None,
                connection_drawing_spec=IRISES_SPEC,
            )

detect(frame)

Find face landmarks and create ROI around the lower face region.

Source code in src\yarppg\roi\facemesh_segmenter.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def detect(self, frame: np.ndarray) -> RegionOfInterest:
    """Find face landmarks and create ROI around the lower face region."""
    rawimg = frame.copy()
    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame)
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        results = self.landmarker.detect_for_video(
            mp_image, int(time.perf_counter() * 1000)
        )

    if len(results.face_landmarks) < 1:
        return RegionOfInterest(np.zeros_like(frame), baseimg=frame)

    if self.draw_landmarks:
        self.draw_facemesh(frame, results.face_landmarks[0], tesselate=True)

    mask, face_rect = self._process_landmarks(frame, results)
    return RegionOfInterest(mask, baseimg=rawimg, face_rect=tuple(face_rect))

draw_facemesh(img, face_landmarks, tesselate=False, contour=False, irises=False)

Draw the detected face landmarks on the image.

Source code in src\yarppg\roi\facemesh_segmenter.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def draw_facemesh(
    self,
    img,
    face_landmarks,
    tesselate=False,
    contour=False,
    irises=False,
):
    """Draw the detected face landmarks on the image."""
    face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()  # type: ignore
    face_landmarks_proto.landmark.extend(
        [
            landmark_pb2.NormalizedLandmark(  # type: ignore
                x=landmark.x, y=landmark.y, z=landmark.z
            )
            for landmark in face_landmarks
        ]
    )
    if tesselate:
        mp.solutions.drawing_utils.draw_landmarks(  # type: ignore
            image=img,
            landmark_list=face_landmarks_proto,
            connections=mp.solutions.face_mesh.FACEMESH_TESSELATION,  # type: ignore
            landmark_drawing_spec=None,
            connection_drawing_spec=TESSELATION_SPEC,
        )
    if contour:
        mp.solutions.drawing_utils.draw_landmarks(  # type: ignore
            image=img,
            landmark_list=face_landmarks_proto,
            connections=mp.solutions.face_mesh.FACEMESH_CONTOURS,  # type: ignore
            landmark_drawing_spec=None,
            connection_drawing_spec=CONTOUR_SPEC,
        )
    if irises:
        mp.solutions.drawing_utils.draw_landmarks(  # type: ignore
            image=img,
            landmark_list=face_landmarks_proto,
            connections=mp.solutions.face_mesh.FACEMESH_IRISES,  # type: ignore
            landmark_drawing_spec=None,
            connection_drawing_spec=IRISES_SPEC,
        )

get_boundingbox_from_coords(coords)

Calculate the bounding rectangle containing all landmarks.

Source code in src\yarppg\roi\facemesh_segmenter.py 
55
56
57
58
59
60
def get_boundingbox_from_coords(coords: np.ndarray) -> np.ndarray:
    """Calculate the bounding rectangle containing all landmarks."""
    xy = np.min(coords, axis=0)
    wh = np.subtract(np.max(coords, axis=0), xy)

    return np.r_[xy, wh]

get_face_landmarker_modelfile()

Get the filename of the FaceLandmarker - download file if necessary.

Source code in src\yarppg\roi\facemesh_segmenter.py 
39
40
41
42
43
44
def get_face_landmarker_modelfile():
    """Get the filename of the FaceLandmarker - download file if necessary."""
    task_filename = "face_landmarker.task"
    return get_cached_resource_path(
        task_filename, MEDIAPIPE_MODELS_BASE + LANDMARKER_TASK
    )

get_landmark_coords(landmarks, width, height)

Extract normalized landmark coordinates to array of pixel coordinates.

Source code in src\yarppg\roi\facemesh_segmenter.py 
47
48
49
50
51
52
def get_landmark_coords(
    landmarks: list[landmark_module.NormalizedLandmark], width: int, height: int
) -> np.ndarray:
    """Extract normalized landmark coordinates to array of pixel coordinates."""
    xyz = [(lm.x, lm.y, lm.z) for lm in landmarks]
    return np.multiply(xyz, [width, height, width]).astype(int)