Using an Accurate Facial Segmentation Model to Detect Skin Tone (Part 2 of 2)
The facial segmentation model provides an effective base on which to develop capabilities for facial attribute prediction. The first facial attribute of interest is skin tone. Previously, in social sciences research that samples colors from facial images, a manual color sampling tool is used to select an approximately 50 pixel x 50 pixel square on either cheek. Needless to say, a significant amount of human judgment is involved in this selection that accounts for image quality, camera angle and other factors that are observed by the researcher selecting the sample. Using facial segmentation, it is now possible to obtain a detailed map of facial features such as Nose, Mouth, Facial Hair, Teeth, etc., which can be used to automate the color sample selection process. The objective of this notebook is to validate the extent to which the automated process can track color estimates selected by a human expert.
- Introduction
- Citations
- Code
- Conclusions
Introduction
Objective
We are building up a set of capabilities to replace the manual work required for researchers and experimenters who want to understand facial characteristics of their subject pool. Those characteristics such as skin tones and facial measurements (e.g. facial width to height ratio) can be used to better understand percieved biases in subjects. What is typically used in statistical studies are self-reported metrics. While self reported metrics are useful to reflect a subject's self identity, they are less useful for systematically reflecting how the world sees that individual. We see a relevant use case in Schniter, et. al. (2020), which examines the pairwise interactions between subjects in a repeated prisoners' dilemma game. In this case, it is useful to understand how each subject is percieved by their partner.
Dataset
The dataset used here was collected in Schniter and Shields, 2020 1. Soellinger, A., Schniter, E. (2020). "Training a UNet for Accurate Facial Attribute Profiling". https://blog.prcvd.ai/research/perception/image/2020/10/12/Training-a-Face-Segmentation-Model-for-Automatic-Skin-Tone-Detection.html.↩ 2. Schniter, E., Shields, T. (2020). "Participant Faces From a Repeated Prisoner’s Dilemma". Unpublished raw data.↩
Citations
%matplotlib inline
from pathlib import Path
# import random
#
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import pandas as pd
import numpy as np
# from PIL import Image
# import imutils
#
#
# from fastai.basics import *
# from fastai.vision import models
# from fastai.vision.all import *
# from fastai.metrics import *
# from fastai.data.all import *
# from fastai.callback import *
#
import skimage
from skimage import color
# from sklearn.mixture import GaussianMixture
#
# import mediapipe as mp
# import cv2
from prcvd.img import (
compute_theta, apply_rgb2lab, calc_euclidean_error, MaskedImg,
LabelMask, FaceMask, MeshMask, TrainedSegmentationModel
)
from prcvd.core import IndexerDict
## shared with training
def get_y_fn(fp):
l_str = str(img_to_l[fp])
out = l_str \
.replace('labels', 'labels_int') \
.replace('png', 'tif')
return out
class FacialProfile:
def __init__(self, img, model, sampling_strategy='use_all', align_face=True):
self.segmask = None
self.eye_slope_threshold = 0.50 #degrees
self.model = model
self.img = img
self.eye_slope = np.nan
self.apply_rotation(angle=0.0)
self.fix_rotation()
self.mesh = MeshMask(img=self.segmask.decoded_img)
self.skin_tones = self.compute_skintones(
sampling_strategy=sampling_strategy
)
self.tot_head_area, self.face_areas = self.create_area_features()
self.bizygomatic_left, self.bizygomatic_right, self.bizygomatic_dist = \
self.compute_bizygomatic_width()
self.upperfacial_top, self.upperfacial_bottom, self.upperfacial_dist = \
self.compute_upperfacial_height()
self.fwhr = self.bizygomatic_dist / self.upperfacial_dist
# self.modeled_img_fp = write_to/'imgs'/(fp.stem+'.jpg') #TODO
def get_profile(self):
"""Returns the face profile object."""
tones = {'rgb_of_{}'.format(k):v for k,v in self.skin_tones.items()}
other = {
'img_rot_degrees': self.img.rotation_degrees,
'img_num_rotations': self.img.num_rotations,
'img_eye_slope': self.eye_slope,
'fwhr': self.fwhr,
'bizygoatic_w_px': self.bizygomatic_dist,
'upperfacial_h_px': self.upperfacial_dist,
'tot_head_area_px': self.tot_head_area
}
out = {**tones, **other}
out = {**out, **self.face_areas}
return out
def apply_rotation(self, angle):
"""Applies a rotation by angle (in degrees) to self.img"""
if angle != 0.0:
self.img.rotate(angle)
self.segmask = FaceMask(img=self.img, model=self.model)
self.create_eye_features()
# TODO: add check to see if the face rotation worked out
self.create_secondary_features()
def fix_rotation(self,):
"""Loops over rotations until the image is corrected."""
while abs(self.theta_degrees) > self.eye_slope_threshold:
self.apply_rotation(angle=self.theta_degrees)
def create_area_features(self):
nots = ['Background/undefined']
total_area = {
lab: np.count_nonzero(self.segmask.mask == code)
for lab, code in self.segmask.label_to_code.items()
if lab not in nots
}
s = sum(total_area.values())
total_area = {'pct_of_head_{}'.format(k): v/s for k,v in total_area.items()}
return s, total_area
def create_eye_features(self):
"""Writes the eye features."""
self.segmask.add_eyes()
self.eye_slope = self.segmask.compute_eye_slope()
if self.eye_slope > 0.0:
self.theta_degrees = \
-1.0*compute_theta(slope1=self.eye_slope, slope2=0.0)
elif self.eye_slope < 0.0:
self.theta_degrees = \
compute_theta(slope2=self.eye_slope, slope1=0.0)
elif self.eye_slope == 0.0:
self.theta_degrees = \
compute_theta(slope2=self.eye_slope, slope1=0.0)
def create_secondary_features(self):
"""Writes the secondary facial features, dependent on reference features."""
self.segmask.add_cheeks()
self.segmask.add_forehead()
def compute_skintones(self, sampling_strategy, thresh=None):
"""Computes skin tones for all regions"""
if thresh:
self.segmask.calc_new_decision(thresh=thresh)
return {
region: self.segmask.calc_region_color(
region=region,
sampling_strategy=sampling_strategy
)
for region in self.segmask.label_to_code.keys()
}
def compute_bizygomatic_width(self):
""""""
# Sources:
# https://carta.anthropogeny.org/moca/topics/upper-facial-height
right_eye_right = self.segmask.find_region_extrema(
region='right_eye',direction='right',
)
right_cheek_right = self.segmask.find_region_extrema(
region='right_cheek',direction='right',
)
bizygomatic_right = (float(right_cheek_right[0]), float(right_eye_right[1]))
left_eye_left = self.segmask.find_region_extrema(
region='left_eye',direction='left',
)
left_cheek_left = self.segmask.find_region_extrema(
region='left_cheek',direction='left',
)
bizygomatic_left = (float(left_cheek_left[0]), float(left_eye_left[1]))
bizygomatic_dist = bizygomatic_right[0] - bizygomatic_left[0]
return bizygomatic_left, bizygomatic_right, bizygomatic_dist
def compute_upperfacial_height(self):
""""""
upperfacial_top = self.segmask.find_region_extrema(
region='Eyebrows',direction='bottom',
)
upperfacial_bottom = self.segmask.find_region_extrema(
region='Lips',direction='top',
)
upperfacial_top = (float(upperfacial_top[0]), float(upperfacial_top[1]))
upperfacial_bottom = (float(upperfacial_bottom[0]), float(upperfacial_bottom[1]))
upperfacial_dist = upperfacial_bottom[1] - upperfacial_top[1]
return upperfacial_top, upperfacial_bottom, upperfacial_dist
def get_skintones_in_lab(self):
"""
Returns skin tones in LAB colors
"""
return {
k: apply_rgb2lab(v)
for k, v in self.skin_tones.items()
}
path = Path("/ws/data/skin-tone/headsegmentation_dataset_ccncsa")
mod_dir = path/'Models'
mod_fp = mod_dir/'checkpoint_20201007'
# TODO: figure out how to get the labels from the trained model...
output_classes = ['Background/undefined', 'Lips', 'Eyes', 'Nose', 'Hair',
'Ears', 'Eyebrows', 'Teeth', 'General face', 'Facial hair',
'Specs/sunglasses']
size = 224
truth_fp = Path('/ws/data/skin-tone/ScreenshotFaceAfterStatement/erics_imgs/MasterFacesDatabaseLabels.xlsx - labels.csv')
truth = pd.read_csv(truth_fp)
truth.index = truth['PostStatementPhotoFilename']
del truth['PostStatementPhotoFilename']
truth['L_m'] = (truth['L_l'] + truth['L_r']) / 2
truth['a_m'] = (truth['a_l'] + truth['a_r']) / 2
truth['b_m'] = (truth['b_l'] + truth['b_r']) / 2
def run_summary(fp, summary, attempt=1, num_attempts=10):
print(fp)
outimg = write_to/'imgs'/(fp.stem+'.jpg')
if outimg.exists():
return None, None
try: del img
except: pass
img = MaskedImg()
img.load_from_file(fn=fp)
try: del model
except: pass
model = TrainedSegmentationModel(
mod_fp=mod_fp,
input_size=size,
output_classes=output_classes
)
try: del profile
except: pass
try:
profile = FacialProfile(
model=model,
img=img,
sampling_strategy='use_all',
align_face=True
)
except:
if not attempt > num_attempts:
return run_summary(
fp=fp,
summary=summary,
attempt=attempt+1,
num_attempts=num_attempts
)
else:
return None, None
plt.figure(figsize=(10,10))
plt.imshow(profile.segmask.decoded_img.img)
plt.imshow(
skimage.color.label2rgb(np.array(profile.segmask.mask)),
alpha=0.3
)
plt.title('Computed fWHR based on Segmentation Only (not FaceMesh).\nfWHR: {}'.format(profile.fwhr))
plt.scatter(x=[profile.bizygomatic_right[0]],
y=[profile.bizygomatic_right[1]],
marker='+', c='orange')
plt.scatter(x=[profile.bizygomatic_left[0]],
y=[profile.bizygomatic_left[1]],
marker='+', c='orange')
plt.plot(
[profile.bizygomatic_right[0], profile.bizygomatic_right[0]],
[0, profile.segmask.mask.shape[1]-1],'ro-')
plt.plot(
[profile.bizygomatic_left[0], profile.bizygomatic_left[0]],
[0, profile.segmask.mask.shape[1]-1],'ro-')
plt.scatter(x=[profile.upperfacial_top[0]],
y=[profile.upperfacial_top[1]],
marker='+', c='red')
plt.plot(
[0, profile.segmask.mask.shape[0]-1],
[profile.upperfacial_top[1], profile.upperfacial_top[1]],
'go-'
)
plt.scatter(x=[profile.upperfacial_bottom[0]],
y=[profile.upperfacial_bottom[1]],
marker='+', c='red')
plt.plot(
[0, profile.segmask.mask.shape[0]-1],
[profile.upperfacial_bottom[1], profile.upperfacial_bottom[1]], 'go-')
plt.savefig(outimg, format='jpeg')
row = profile.get_profile()
row['model_id'] = mod_fp
return row, plt
erics_img = Path('/ws/data/skin-tone/from_zenodo/Media/MediaForExport/')
ls = [fp for fp in list(erics_img.ls()) if str(fp)[-4:] == '.jpg']
# row = truth.loc[fp.name].to_dict()
write_to = Path('/ws/data/skin-tone/output1')
summary = []
fp = ls[1]
outimg = write_to/'imgs'/(fp.stem+'.jpg')
try: del img
except: pass
img = MaskedImg()
img.load_from_file(fn=fp)
try: del model
except: pass
model = TrainedSegmentationModel(
mod_fp=mod_fp,
input_size=size,
output_classes=output_classes
)
try: del profile
except: pass
profile = FacialProfile(
model=model,
img=img,
sampling_strategy='use_all',
align_face=True
)
plt.imshow(profile.img.img)
print(profile.bizygomatic_right)
profile.bizygomatic_left
def plot_all(save=False)
plt.figure(figsize=(10,10))
plt.imshow(profile.segmask.decoded_img.img)
plt.imshow(
skimage.color.label2rgb(np.array(profile.segmask.mask)),
alpha=0.3
)
plt.title('Computed fWHR based on Segmentation Only (not FaceMesh).\nfWHR: {}'.format(profile.fwhr))
plt.scatter(x=[profile.bizygomatic_right[0]],
y=[profile.bizygomatic_right[1]],
marker='+', c='orange')
plt.scatter(x=[profile.bizygomatic_left[0]],
y=[profile.bizygomatic_left[1]],
marker='+', c='orange')
plt.plot(
[profile.bizygomatic_right[0], profile.bizygomatic_right[0]],
[0, profile.segmask.mask.shape[1]-1],'ro-')
plt.plot(
[profile.bizygomatic_left[0], profile.bizygomatic_left[0]],
[0, profile.segmask.mask.shape[1]-1],'ro-')
plt.scatter(x=[profile.upperfacial_top[0]],
y=[profile.upperfacial_top[1]],
marker='+', c='red')
plt.plot(
[0, profile.segmask.mask.shape[0]-1],
[profile.upperfacial_top[1], profile.upperfacial_top[1]],
'go-'
)
plt.scatter(x=[profile.upperfacial_bottom[0]],
y=[profile.upperfacial_bottom[1]],
marker='+', c='red')
plt.plot(
[0, profile.segmask.mask.shape[0]-1],
[profile.upperfacial_bottom[1], profile.upperfacial_bottom[1]], 'go-')
plt.savefig(outimg, format='jpeg')
row = profile.get_profile()
row['model_id'] = mod_fp
##Exceptions
# ZeroDivisionError, RuntimeError
outimg
summary = []
failures = []
for fp in ls[:10]:
if str(fp)[-4:] != '.jpg': continue
try:
row, plt = run_summary(fp=fp, summary=summary)
except:
failures.append(fp)
continue
if not row and not plt:
failures.append(fp)
continue
summary.append(row)
failures
profile.get_profile()
plt.figure(figsize = (8,8))
plt.title('Original Image, Rotated and Shrunken to size={}'.format(size))
plt.imshow(profile.segmask.decoded_img.img)
plt.figure(figsize = (8,8))
plt.title('Enhanced Segmentation Model Output')
plt.imshow(profile.segmask.mask)
profile.mesh.draw(thickness=1, circle_radius=1)
plt.figure(figsize=(10,10))
plt.imshow(profile.segmask.decoded_img.img)
plt.imshow(
skimage.color.label2rgb(np.array(profile.segmask.mask)),
alpha=0.3
)
plt.title('Computed fWHR based on Segmentation Only (not FaceMesh).\nfWHR: {}'.format(profile.fwhr))
plt.scatter(x=[profile.bizygomatic_right[0]],
y=[profile.bizygomatic_right[1]],
marker='+', c='orange')
plt.scatter(x=[profile.bizygomatic_left[0]],
y=[profile.bizygomatic_left[1]],
marker='+', c='orange')
plt.plot(
[profile.bizygomatic_right[0], profile.bizygomatic_right[0]],
[0, profile.segmask.mask.shape[1]-1],'ro-')
plt.plot(
[profile.bizygomatic_left[0], profile.bizygomatic_left[0]],
[0, profile.segmask.mask.shape[1]-1],'ro-')
plt.scatter(x=[profile.upperfacial_top[0]],
y=[profile.upperfacial_top[1]],
marker='+', c='red')
plt.plot(
[0, profile.segmask.mask.shape[0]-1],
[profile.upperfacial_top[1], profile.upperfacial_top[1]],
'go-'
)
plt.scatter(x=[profile.upperfacial_bottom[0]],
y=[profile.upperfacial_bottom[1]],
marker='+', c='red')
plt.plot(
[0, profile.segmask.mask.shape[0]-1],
[profile.upperfacial_bottom[1], profile.upperfacial_bottom[1]], 'go-')