import colorsys
import math
from enum import IntEnum
import cv2
import numpy as np
import numpy.linalg as npla
from face_detect.core import imagelib
from face_detect.core import mathlib
from face_detect.core.mathlib.umeyama import umeyama
from face_detect.FaceType import FaceType
mesh_33=[70,63,105,66,107,336,296,334,293,300,168,197,5,4,240,99,2,328,460,33,160,158,133,153,144,362,385,387,263,373,380,57,287]
landmarks_2D_4=np.array([
[0.224152 , 0.2119465], #left iris mean 37 38 40 41
[0.75610125, 0.2119465],#right iris mean 43 44 46 47
[0.490127, 0.515625], # nose 30
[0.4901265, 0.780233] #mouth mean 48 54
])
landmarks_2D_4_bottom=np.array([
[0.2218305, 0.244588 ], #left iris mean 40 41
[0.7584225, 0.244588],#right iris mean 46 47
[0.490127, 0.515625], # nose 30
[0.4901265, 0.780233] #mouth mean 48 54
])
landmarks_2D = np.array([
[0.000213256, 0.106454], # 17
[0.0752622, 0.038915], # 18
[0.18113, 0.0187482], # 19
[0.29077, 0.0344891], # 20
[0.393397, 0.0773906], # 21
[0.586856, 0.0773906], # 22
[0.689483, 0.0344891], # 23
[0.799124, 0.0187482], # 24
[0.904991, 0.038915], # 25
[0.98004, 0.106454], # 26
[0.490127, 0.203352], # 27
[0.490127, 0.307009], # 28
[0.490127, 0.409805], # 29
[0.490127, 0.515625], # 30
[0.36688, 0.587326], # 31
[0.426036, 0.609345], # 32
[0.490127, 0.628106], # 33
[0.554217, 0.609345], # 34
[0.613373, 0.587326], # 35
[0.121737, 0.216423], # 36
[0.187122, 0.178758], # 37
[0.265825, 0.179852], # 38
[0.334606, 0.231733], # 39
[0.260918, 0.245099], # 40
[0.182743, 0.244077], # 41
[0.645647, 0.231733], # 42
[0.714428, 0.179852], # 43
[0.793132, 0.178758], # 44
[0.858516, 0.216423], # 45
[0.79751, 0.244077], # 46
[0.719335, 0.245099], # 47
[0.254149, 0.780233], # 48
[0.340985, 0.745405], # 49
[0.428858, 0.727388], # 50
[0.490127, 0.742578], # 51
[0.551395, 0.727388], # 52
[0.639268, 0.745405], # 53
[0.726104, 0.780233], # 54
[0.642159, 0.864805], # 55
[0.556721, 0.902192], # 56
[0.490127, 0.909281], # 57
[0.423532, 0.902192], # 58
[0.338094, 0.864805], # 59
[0.290379, 0.784792], # 60
[0.428096, 0.778746], # 61
[0.490127, 0.785343], # 62
[0.552157, 0.778746], # 63
[0.689874, 0.784792], # 64
[0.553364, 0.824182], # 65
[0.490127, 0.831803], # 66
[0.42689, 0.824182] # 67
], dtype=np.float32)
landmarks_2D_new = np.array([
[0.000213256, 0.106454], # 17
[0.0752622, 0.038915], # 18
[0.18113, 0.0187482], # 19
[0.29077, 0.0344891], # 20
[0.393397, 0.0773906], # 21
[0.586856, 0.0773906], # 22
[0.689483, 0.0344891], # 23
[0.799124, 0.0187482], # 24
[0.904991, 0.038915], # 25
[0.98004, 0.106454], # 26
[0.490127, 0.203352], # 27
[0.490127, 0.307009], # 28
[0.490127, 0.409805], # 29
[0.490127, 0.515625], # 30
[0.36688, 0.587326], # 31
[0.426036, 0.609345], # 32
[0.490127, 0.628106], # 33
[0.554217, 0.609345], # 34
[0.613373, 0.587326], # 35
[0.121737, 0.216423], # 36
[0.187122, 0.178758], # 37
[0.265825, 0.179852], # 38
[0.334606, 0.231733], # 39
[0.260918, 0.245099], # 40
[0.182743, 0.244077], # 41
[0.645647, 0.231733], # 42
[0.714428, 0.179852], # 43
[0.793132, 0.178758], # 44
[0.858516, 0.216423], # 45
[0.79751, 0.244077], # 46
[0.719335, 0.245099], # 47
[0.254149, 0.780233], # 48
[0.726104, 0.780233], # 54
], dtype=np.float32)
landmarks_2D_new_mesh = np.array([
[ 0.000213256, 0.106454 ], #17
[ 0.0752622, 0.038915 ], #18
[0.1281961, 0.0288316], #19[ 0.18113, 0.0187482 ]
[ 0.29077, 0.0144891 ], #20
[ 0.393397, 0.0773906 ], #21
[ 0.586856, 0.0773906 ], #22
[ 0.689483, 0.0144891 ], #23
[0.8520575, 0.0288316], #24[ 0.799124, 0.0187482 ]
[ 0.904991, 0.038915 ], #25
[ 0.98004, 0.106454 ], #26
[ 0.490127, 0.203352 ], #27
[ 0.490127, 0.307009 ], #28
[ 0.490127, 0.409805 ], #29
[ 0.490127, 0.515625 ], #30
[0.396458 , 0.5983355], #31 [ 0.36688, 0.587326 ]
[ 0.426036, 0.609345 ], #32
[ 0.490127, 0.628106 ], #33
[ 0.554217, 0.609345 ], #34
[ 0.613373, 0.587326 ], #35
[ 0.071737, 0.136423 ], #36
[ 0.137122, 0.118758 ], #37
[ 0.215825, 0.119852 ], #38
[ 0.334606, 0.151733 ], #39
[ 0.210918, 0.165099 ], #40
[ 0.132743, 0.164077 ], #41
[ 0.645647, 0.151733 ], #42
[ 0.764428, 0.119852 ], #43
[ 0.743132, 0.118758 ], #44
[ 0.908516, 0.136423 ], #45
[ 0.84751, 0.164077 ], #46
[ 0.769335, 0.165099 ], #47
[ 0.254149, 0.780233 ], #48
[ 0.726104, 0.780233 ], #54
], dtype=np.float32)
# landmarks_468_moving_parts_indexes = [0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 40, 41, 42, 43, 46, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 76, 77, 78, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 95, 96, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 117, 118, 124, 130, 132, 133, 135, 136, 138, 139, 140, 143, 144, 145, 146, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 168, 169, 170, 171, 172, 173, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 189, 190, 191, 192, 193, 194, 199, 200, 201, 202, 204, 208, 210, 211, 212, 213, 214, 215, 221, 222, 223, 224, 225, 226, 228, 229, 230, 231, 232, 233, 243, 244, 245, 246, 247, 249, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 267, 268, 269, 270, 271, 272, 273, 276, 282, 283, 284, 285, 286, 287, 288, 291, 292, 293, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 306, 307, 308, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 324, 325, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 346, 347, 353, 359, 361, 362, 364, 365, 367, 368, 369, 372, 373, 374, 375, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 394, 395, 396, 397, 398, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 413, 414, 415, 416, 417, 418, 421, 422, 424, 428, 430, 431, 432, 433, 434, 435, 441, 442, 443, 444, 445, 446, 448, 449, 450, 451, 452, 453, 463, 464, 465, 466, 467]
# uni_landmarks_468 = np.array(
# [[ 0.49066195, 0.7133885 ],
# [ 0.49042386, 0.52723485],
# [ 0.49050152, 0.6244965 ],
# [ 0.45844677, 0.39348277],
# [ 0.4905825 , 0.49120593],
# [ 0.49006602, 0.43998772],
# [ 0.48907965, 0.26775706],
# [ 0.11721139, 0.23243594],
# [ 0.48957095, 0.11063451],
# [ 0.48949632, 0.03535742],
# [ 0.48905632, -0.25326234],
# [ 0.4907858 , 0.73766613],
# [ 0.49081355, 0.7606857 ],
# [ 0.4908666 , 0.7839426 ],
# [ 0.49079415, 0.78913504],
# [ 0.4908271 , 0.80801845],
# [ 0.49086872, 0.831855 ],
# [ 0.49092326, 0.8631041 ],
# [ 0.49104446, 0.94170016],
# [ 0.49009967, 0.5546924 ],
# [ 0.44398275, 0.5741402 ],
# [-0.2106727 , 0.00861922],
# [ 0.2523662 , 0.2832579 ],
# [ 0.2042254 , 0.28945392],
# [ 0.1552372 , 0.28322184],
# [ 0.09056008, 0.24730967],
# [ 0.30096018, 0.27277085],
# [ 0.21548809, 0.16713436],
# [ 0.2595488 , 0.17071684],
# [ 0.16957955, 0.17298089],
# [ 0.13164258, 0.18425746],
# [ 0.043018 , 0.28581 ],
# [ 0.30856833, 1.0507976 ],
# [ 0.10015843, 0.22331452],
# [-0.20773543, 0.26701325],
# [-0.02414621, 0.25144747],
# [ 0.23481508, 0.5045001 ],
# [ 0.44063616, 0.7097012 ],
# [ 0.4449884 , 0.762481 ],
# [ 0.3840104 , 0.7218947 ],
# [ 0.33943903, 0.73847425],
# [ 0.40284824, 0.76374006],
# [ 0.36457124, 0.76704985],
# [ 0.26937196, 0.84716266],
# [ 0.46683946, 0.5275276 ],
# [ 0.4642676 , 0.49167544],
# [ 0.06039319, 0.11509081],
# [ 0.31504983, 0.36394927],
# [ 0.3660137 , 0.52945083],
# [ 0.3509634 , 0.50311893],
# [ 0.09496811, 0.5005815 ],
# [ 0.46075967, 0.4424029 ],
# [ 0.20108324, 0.05883435],
# [ 0.12877828, 0.07731954],
# [-0.09675749, -0.09848522],
# [ 0.39672711, 0.09345116],
# [ 0.29908365, 0.18449144],
# [ 0.23298171, 0.7922538 ],
# [-0.27583498, 0.85219014],
# [ 0.38898414, 0.5723152 ],
# [ 0.41446668, 0.59347576],
# [ 0.28167963, 0.7884952 ],
# [ 0.30013445, 0.7875627 ],
# [ 0.09448256, 0.03961415],
# [ 0.3531811 , 0.5553779 ],
# [ 0.2873921 , 0.05599196],
# [ 0.28232294, 0.01076962],
# [ 0.1903341 , -0.23029903],
# [ 0.0108011 , -0.03099815],
# [ 0.24915197, -0.10741784],
# [ 0.01047484, 0.08868673],
# [-0.08942058, 0.05201372],
# [ 0.44268388, 0.7376863 ],
# [ 0.39652622, 0.741894 ],
# [ 0.35389552, 0.7514722 ],
# [ 0.393559 , 0.5851372 ],
# [ 0.2925385 , 0.7871472 ],
# [ 0.31904542, 0.80939215],
# [ 0.32005206, 0.787085 ],
# [ 0.4195982 , 0.5444628 ],
# [ 0.3688312 , 0.78418756],
# [ 0.40608776, 0.7841225 ],
# [ 0.4472093 , 0.78405076],
# [ 0.43053833, 0.9379409 ],
# [ 0.44192585, 0.8617842 ],
# [ 0.44321233, 0.82923037],
# [ 0.4432334 , 0.80578357],
# [ 0.44304678, 0.78921837],
# [ 0.36314115, 0.7893578 ],
# [ 0.36057413, 0.8040033 ],
# [ 0.35472178, 0.8187327 ],
# [ 0.34614718, 0.83330894],
# [ 0.2959003 , 0.69076014],
# [-0.37090415, 0.5509728 ],
# [ 0.4903264 , 0.5851119 ],
# [ 0.3370172 , 0.78961957],
# [ 0.33070365, 0.8010128 ],
# [ 0.43397966, 0.6231119 ],
# [ 0.35356513, 0.59569615],
# [ 0.42509514, 0.6093918 ],
# [ 0.2635329 , 0.39636588],
# [ 0.19704658, 0.43663597],
# [ 0.33384863, 0.52658314],
# [ 0.03225203, -0.18047164],
# [ 0.11854403, -0.08533629],
# [ 0.18350407, 0.01215954],
# [ 0.31292278, 0.8845064 ],
# [ 0.3862302 , 0.02093028],
# [ 0.36480215, -0.1098879 ],
# [ 0.33342764, -0.2497105 ],
# [ 0.11592615, 0.2646692 ],
# [-0.00803981, 0.3294946 ],
# [ 0.33535972, 0.26431814],
# [ 0.05940344, 0.18766014],
# [ 0.36188984, 0.33336782],
# [ 0.39879864, 0.50869733],
# [-0.07952328, 0.36885905],
# [ 0.04230375, 0.36800843],
# [ 0.11137532, 0.3864613 ],
# [ 0.19386435, 0.37397826],
# [ 0.25749052, 0.34993485],
# [ 0.310977 , 0.3240539 ],
# [ 0.44813582, 0.2762354 ],
# [-0.06039021, 0.4864401 ],
# [ 0.00945808, 0.17624807],
# [ 0.4739895 , 0.55369264],
# [ 0.32125092, 0.4170324 ],
# [-0.36162117, 0.27013144],
# [ 0.3592803 , 0.3023075 ],
# [ 0.30784345, 0.529875 ],
# [ 0.07601253, 0.22579695],
# [ 0.3824061 , 0.47686696],
# [-0.33810768, 0.70034444],
# [ 0.34643772, 0.24336138],
# [ 0.42429656, 0.45338264],
# [ 0.02854156, 0.939626 ],
# [-0.04352415, 1.0322431 ],
# [-0.20510256, 0.51651907],
# [-0.06969981, 0.8698207 ],
# [-0.1581445 , 0.14948419],
# [ 0.2889787 , 1.1224228 ],
# [ 0.47446907, 0.58377683],
# [ 0.2818322 , 0.4586393 ],
# [-0.08708218, 0.2627534 ],
# [ 0.16877942, 0.25976214],
# [ 0.21234928, 0.267416 ],
# [ 0.30676025, 0.81592965],
# [-0.06259334, 0.6009466 ],
# [ 0.36930662, 1.2302231 ],
# [ 0.17070079, 1.149443 ],
# [ 0.07714309, 1.0989524 ],
# [ 0.48931465, -0.1052461 ],
# [ 0.49159575, 1.2484183 ],
# [ 0.2527582 , 0.26420003],
# [ 0.30066028, 0.25829503],
# [ 0.3310663 , 0.25034374],
# [-0.05075949, 0.16421606],
# [ 0.29250854, 0.19938153],
# [ 0.2522571 , 0.18826446],
# [ 0.21220936, 0.18724632],
# [ 0.16866222, 0.19260857],
# [ 0.13789575, 0.2011967 ],
# [-0.29335994, 0.12383505],
# [ 0.1379709 , 0.24424627],
# [ 0.49057597, 0.65296 ],
# [ 0.34147182, 0.663431 ],
# [ 0.3941785 , 0.5603462 ],
# [ 0.43007633, 0.6569765 ],
# [ 0.48963526, 0.17996965],
# [ 0.11681002, 1.0107123 ],
# [ 0.19942053, 1.068824 ],
# [ 0.38605705, 1.1563928 ],
# [-0.16756529, 0.9615808 ],
# [ 0.32817602, 0.21989337],
# [ 0.41141313, 0.3578073 ],
# [ 0.49127796, 1.1678538 ],
# [ 0.27080515, 1.195178 ],
# [-0.19307071, 0.6481067 ],
# [ 0.399859 , 0.7892937 ],
# [ 0.39875022, 0.80587196],
# [ 0.39717573, 0.8256797 ],
# [ 0.3931817 , 0.85224336],
# [ 0.3670306 , 0.9161113 ],
# [ 0.3256227 , 0.7724022 ],
# [ 0.31488904, 0.76426226],
# [ 0.3001029 , 0.7583232 ],
# [ 0.2565659 , 0.73397243],
# [ 0.0438394 , 0.6234349 ],
# [ 0.40628996, 0.30296788],
# [ 0.37707803, 0.19498621],
# [ 0.34125936, 0.21069102],
# [ 0.33733743, 0.7842425 ],
# [ 0.00882016, 0.769232 ],
# [ 0.4335431 , 0.1821002 ],
# [ 0.33409703, 0.9826546 ],
# [ 0.49011812, 0.3896104 ],
# [ 0.45311242, 0.34152514],
# [ 0.4899982 , 0.33611432],
# [ 0.369907 , 0.43193236],
# [ 0.49116373, 1.0932964 ],
# [ 0.49107185, 1.0132186 ],
# [ 0.41421878, 1.008873 ],
# [ 0.21551576, 0.8785059 ],
# [ 0.27587482, 0.57461077],
# [ 0.2683325 , 0.9399872 ],
# [ 0.17091931, 0.56899554],
# [ 0.23741819, 0.6283017 ],
# [ 0.12783033, 0.65916985],
# [ 0.39875996, 1.0855893 ],
# [ 0.33251646, 0.45881665],
# [ 0.16138549, 0.93153137],
# [ 0.23269826, 0.99740875],
# [ 0.17994387, 0.8051213 ],
# [-0.06026869, 0.7033027 ],
# [ 0.10063827, 0.8241594 ],
# [-0.15810522, 0.7679798 ],
# [ 0.2014156 , 0.7000692 ],
# [ 0.365875 , 0.3839739 ],
# [ 0.4115726 , 0.5293855 ],
# [ 0.378973 , 0.5476473 ],
# [ 0.43235463, 0.49621448],
# [ 0.3385827 , 0.15134089],
# [ 0.27179635, 0.12940899],
# [ 0.21341887, 0.12485553],
# [ 0.15807948, 0.12881717],
# [ 0.10610204, 0.14814937],
# [ 0.03133116, 0.236169 ],
# [-0.21341309, 0.38895622],
# [ 0.07818349, 0.3101151 ],
# [ 0.1318462 , 0.32528982],
# [ 0.19485526, 0.32642388],
# [ 0.25329807, 0.31256682],
# [ 0.30569646, 0.29578218],
# [ 0.34839994, 0.2842457 ],
# [-0.3824783 , 0.41054142],
# [ 0.37162504, 0.5664833 ],
# [ 0.41687053, 0.40615496],
# [ 0.4433516 , 0.5242282 ],
# [ 0.44805393, 0.5562703 ],
# [ 0.43453053, 0.5407472 ],
# [ 0.37351128, 0.58924097],
# [ 0.46121803, 0.55474806],
# [ 0.45942986, 0.5810936 ],
# [ 0.35955238, 0.24802393],
# [ 0.38181108, 0.25985107],
# [ 0.40143687, 0.26679716],
# [ 0.11717269, 0.2102652 ],
# [ 0.0940459 , 0.2016577 ],
# [ 0.5217974 , 0.39331725],
# [ 0.8625129 , 0.23113514],
# [ 0.5369363 , 0.57397795],
# [ 1.1896138 , 0.00617525],
# [ 0.7275363 , 0.28242856],
# [ 0.7756985 , 0.2884565 ],
# [ 0.82466465, 0.28205347],
# [ 0.88921595, 0.24591576],
# [ 0.6788919 , 0.27210945],
# [ 0.7640089 , 0.166177 ],
# [ 0.7199609 , 0.16991326],
# [ 0.8099376 , 0.17186326],
# [ 0.8479136 , 0.18300733],
# [ 0.9368992 , 0.28424102],
# [ 0.67367214, 1.0503516 ],
# [ 0.8795338 , 0.22195426],
# [ 1.1875838 , 0.26458502],
# [ 1.0039485 , 0.24965489],
# [ 0.74551606, 0.50375396],
# [ 0.54075617, 0.7095265 ],
# [ 0.5365969 , 0.76231945],
# [ 0.59742403, 0.7215222 ],
# [ 0.6420548 , 0.7379461 ],
# [ 0.5787324 , 0.7634331 ],
# [ 0.617019 , 0.766611 ],
# [ 0.71218634, 0.8469107 ],
# [ 0.513503 , 0.52683127],
# [ 0.5170686 , 0.49132976],
# [ 0.91894245, 0.11362247],
# [ 0.66487545, 0.36299667],
# [ 0.61502695, 0.52894545],
# [ 0.6296784 , 0.50242335],
# [ 0.88566196, 0.49919614],
# [ 0.5193738 , 0.4423927 ],
# [ 0.7780587 , 0.05788935],
# [ 0.8504331 , 0.07610969],
# [ 1.0753254 , -0.1005309 ],
# [ 0.5824533 , 0.09305263],
# [ 0.6804744 , 0.18382579],
# [ 0.7485537 , 0.79121745],
# [ 1.2577202 , 0.8495136 ],
# [ 0.59192824, 0.57196105],
# [ 0.5665197 , 0.59321034],
# [ 0.6999867 , 0.7877651 ],
# [ 0.6814933 , 0.7868972 ],
# [ 0.8846023 , 0.03829005],
# [ 0.62761134, 0.5547819 ],
# [ 0.6917209 , 0.05532694],
# [ 0.6966465 , 0.01012804],
# [ 0.7876697 , -0.2309872 ],
# [ 0.9680314 , -0.03263693],
# [ 0.7294528 , -0.1080169 ],
# [ 0.96877015, 0.08704082],
# [ 1.0685298 , 0.05000517],
# [ 0.538806 , 0.7375185 ],
# [ 0.5849781 , 0.7415651 ],
# [ 0.62764204, 0.7509944 ],
# [ 0.58739805, 0.5847989 ],
# [ 0.68912315, 0.78645504],
# [ 0.6626941 , 0.8087924 ],
# [ 0.6616096 , 0.7864889 ],
# [ 0.5612171 , 0.5442156 ],
# [ 0.61282057, 0.7837617 ],
# [ 0.575564 , 0.7838267 ],
# [ 0.5344426 , 0.7838985 ],
# [ 0.551505 , 0.93764293],
# [ 0.5399973 , 0.8616131 ],
# [ 0.53859717, 0.8290639 ],
# [ 0.5384943 , 0.8056173 ],
# [ 0.53862303, 0.78905153],
# [ 0.6185288 , 0.78891206],
# [ 0.62114686, 0.8035485 ],
# [ 0.62705064, 0.81825733],
# [ 0.635676 , 0.8328036 ],
# [ 0.6854969 , 0.69067734],
# [ 1.3517375 , 0.54796624],
# [ 0.64465326, 0.78908265],
# [ 0.6510032 , 0.8004538 ],
# [ 0.5471015 , 0.62291807],
# [ 0.62742317, 0.59512955],
# [ 0.55593795, 0.6091671 ],
# [ 0.7161671 , 0.39546603],
# [ 0.7836529 , 0.435396 ],
# [ 0.64694774, 0.5258542 ],
# [ 0.94603044, -0.1820665 ],
# [ 0.86011904, -0.08652072],
# [ 0.79549086, 0.01118712],
# [ 0.66893554, 0.8840338 ],
# [ 0.59274685, 0.02056277],
# [ 0.613851 , -0.11025709],
# [ 0.64526045, -0.25000137],
# [ 0.8639107 , 0.26336375],
# [ 0.9881146 , 0.3277454 ],
# [ 0.6445285 , 0.26371115],
# [ 0.92017305, 0.18616839],
# [ 0.61790556, 0.3323734 ],
# [ 0.58225924, 0.5077285 ],
# [ 1.0597262 , 0.36687428],
# [ 0.93791103, 0.36642405],
# [ 0.86892897, 0.38505408],
# [ 0.78624976, 0.37287512],
# [ 0.7223912 , 0.34902957],
# [ 0.6687594 , 0.32310694],
# [ 0.5315497 , 0.2757726 ],
# [ 1.0409807 , 0.48452145],
# [ 0.9700836 , 0.17458573],
# [ 0.5065989 , 0.55419755],
# [ 0.6590531 , 0.41624966],
# [ 1.3414742 , 0.26715896],
# [ 0.62023264, 0.30108824],
# [ 0.67289865, 0.5290446 ],
# [ 0.9036883 , 0.22435239],
# [ 0.59769833, 0.47659585],
# [ 1.3194624 , 0.6974514 ],
# [ 0.63339525, 0.24286939],
# [ 0.5571053 , 0.45250946],
# [ 0.9535533 , 0.9380257 ],
# [ 1.0260391 , 1.0303764 ],
# [ 1.1858007 , 0.51410204],
# [ 1.0515786 , 0.867869 ],
# [ 1.1375865 , 0.14722979],
# [ 0.6935665 , 1.1218798 ],
# [ 0.5063422 , 0.58382744],
# [ 0.69926125, 0.45745537],
# [ 1.0669235 , 0.26074636],
# [ 0.8110406 , 0.25864118],
# [ 0.7674977 , 0.26644707],
# [ 0.67500204, 0.81528693],
# [ 1.0435516 , 0.5990178 ],
# [ 0.6121316 , 1.2306852 ],
# [ 0.81222653, 1.1483234 ],
# [ 0.9056057 , 1.0975065 ],
# [ 0.7270778 , 0.26337218],
# [ 0.6791554 , 0.25763443],
# [ 0.6487802 , 0.24975733],
# [ 1.0302606 , 0.16233999],
# [ 0.68710136, 0.19869283],
# [ 0.72731376, 0.18743533],
# [ 0.7673578 , 0.1862774 ],
# [ 0.81092334, 0.1914876 ],
# [ 0.84171957, 0.1999683 ],
# [ 1.2727026 , 0.12110176],
# [ 0.8417947 , 0.24301787],
# [ 0.63978463, 0.6627527 ],
# [ 0.5866921 , 0.5600102 ],
# [ 0.5511283 , 0.6567636 ],
# [ 0.8655194 , 1.009457 ],
# [ 0.78306264, 1.0678959 ],
# [ 0.59620714, 1.1564037 ],
# [ 1.149833 , 0.9592815 ],
# [ 0.65151644, 0.21932903],
# [ 0.56865776, 0.3571483 ],
# [ 0.71228063, 1.1944076 ],
# [ 1.1742088 , 0.6457327 ],
# [ 0.5818109 , 0.78897613],
# [ 0.5829775 , 0.80555046],
# [ 0.5846211 , 0.82535255],
# [ 0.5887078 , 0.8519021 ],
# [ 0.6150045 , 0.916079 ],
# [ 0.65597004, 0.771831 ],
# [ 0.66669285, 0.7636482 ],
# [ 0.6814582 , 0.7576576 ],
# [ 0.7245435 , 0.73241323],
# [ 0.9371713 , 0.62184393],
# [ 0.5736738 , 0.30186948],
# [ 0.60240346, 0.19448838],
# [ 0.6383993 , 0.21017241],
# [ 0.64431435, 0.7837067 ],
# [ 0.9726586 , 0.7675604 ],
# [ 0.54576766, 0.18157108],
# [ 0.6477745 , 0.98230904],
# [ 0.5269076 , 0.34123868],
# [ 0.61068684, 0.43131724],
# [ 0.56792 , 1.0087004 ],
# [ 0.7662271 , 0.8776794 ],
# [ 0.7048996 , 0.57387614],
# [ 0.7136024 , 0.9394351 ],
# [ 0.8097781 , 0.56784695],
# [ 0.7435453 , 0.62753886],
# [ 0.85328954, 0.6578133 ],
# [ 0.5835228 , 1.0854707 ],
# [ 0.64810187, 0.45811343],
# [ 0.82059515, 0.9304676 ],
# [ 0.7494546 , 0.9966611 ],
# [ 0.8015866 , 0.80400985],
# [ 1.0415541 , 0.70138854],
# [ 0.8809724 , 0.8228132 ],
# [ 1.1396528 , 0.7657218 ],
# [ 0.7798614 , 0.69881856],
# [ 0.6143189 , 0.383193 ],
# [ 0.56934875, 0.52867246],
# [ 0.60162777, 0.54706186],
# [ 0.5470082 , 0.4963955 ],
# [ 0.6408297 , 0.15073723],
# [ 0.7075675 , 0.12865019],
# [ 0.76593757, 0.12391254],
# [ 0.8212976 , 0.12768434],
# [ 0.87334216, 0.14682971],
# [ 0.948411 , 0.23457018],
# [ 1.1936799 , 0.38651106],
# [ 0.90181875, 0.30865455],
# [ 0.84818983, 0.3240165 ],
# [ 0.7851249 , 0.32537246],
# [ 0.72658616, 0.3116911 ],
# [ 0.6740513 , 0.2949461 ],
# [ 0.63111407, 0.28325075],
# [ 1.362823 , 0.4074953 ],
# [ 0.60951644, 0.5658945 ],
# [ 0.5634702 , 0.4055624 ],
# [ 0.5374476 , 0.5247268 ],
# [ 0.53280455, 0.5561224 ],
# [ 0.5462737 , 0.5405522 ],
# [ 0.6075077 , 0.58877414],
# [ 0.51933056, 0.55477065],
# [ 0.52143395, 0.58103496],
# [ 0.62030756, 0.24758299],
# [ 0.59746987, 0.2574137 ],
# [ 0.5780933 , 0.2652785 ],
# [ 0.8624742 , 0.2089644 ],
# [ 0.8855709 , 0.20027623]], dtype=np.float32)
# mesh_33 = np.arange(468)
# mask = np.ones(len(mesh_33), dtype=bool)
# mask[landmarks_468_moving_parts_indexes]=False
# mesh_33=mesh_33[mask,...]
# landmarks_2D_new_mesh=uni_landmarks_468[mask,...]
# mouth_center_landmarks_2D = np.array([
# [-4.4202591e-07, 4.4916576e-01], # 48
# [1.8399176e-01, 3.7537053e-01], # 49
# [3.7018123e-01, 3.3719531e-01], # 50
# [5.0000089e-01, 3.6938059e-01], # 51
# [6.2981832e-01, 3.3719531e-01], # 52
# [8.1600773e-01, 3.7537053e-01], # 53
# [1.0000000e+00, 4.4916576e-01], # 54
# [8.2213330e-01, 6.2836081e-01], # 55
# [6.4110327e-01, 7.0757812e-01], # 56
# [5.0000089e-01, 7.2259867e-01], # 57
# [3.5889623e-01, 7.0757812e-01], # 58
# [1.7786618e-01, 6.2836081e-01], # 59
# [7.6765373e-02, 4.5882553e-01], # 60
# [3.6856663e-01, 4.4601500e-01], # 61
# [5.0000089e-01, 4.5999300e-01], # 62
# [6.3143289e-01, 4.4601500e-01], # 63
# [9.2323411e-01, 4.5882553e-01], # 64
# [6.3399029e-01, 5.4228687e-01], # 65
# [5.0000089e-01, 5.5843467e-01], # 66
# [3.6601129e-01, 5.4228687e-01] # 67
# ], dtype=np.float32)
# 68 point landmark definitions
landmarks_68_pt = {"mouth": (48, 68),
"right_eyebrow": (17, 22),
"left_eyebrow": (22, 27),
"right_eye": (36, 42),
"left_eye": (42, 48),
"nose": (27, 36), # missed one point
"jaw": (0, 17)}
landmarks_68_3D = np.array([
[-73.393523, -29.801432, 47.667532], # 00
[-72.775014, -10.949766, 45.909403], # 01
[-70.533638, 7.929818, 44.842580], # 02
[-66.850058, 26.074280, 43.141114], # 03
[-59.790187, 42.564390, 38.635298], # 04
[-48.368973, 56.481080, 30.750622], # 05
[-34.121101, 67.246992, 18.456453], # 06
[-17.875411, 75.056892, 3.609035], # 07
[0.098749, 77.061286, -0.881698], # 08
[17.477031, 74.758448, 5.181201], # 09
[32.648966, 66.929021, 19.176563], # 10
[46.372358, 56.311389, 30.770570], # 11
[57.343480, 42.419126, 37.628629], # 12
[64.388482, 25.455880, 40.886309], # 13
[68.212038, 6.990805, 42.281449], # 14
[70.486405, -11.666193, 44.142567], # 15
[71.375822, -30.365191, 47.140426], # 16
[-61.119406, -49.361602, 14.254422], # 17
[-51.287588, -58.769795, 7.268147], # 18
[-37.804800, -61.996155, 0.442051], # 19
[-24.022754, -61.033399, -6.606501], # 20
[-11.635713, -56.686759, -11.967398], # 21
[12.056636, -57.391033, -12.051204], # 22
[25.106256, -61.902186, -7.315098], # 23
[38.338588, -62.777713, -1.022953], # 24
[51.191007, -59.302347, 5.349435], # 25
[60.053851, -50.190255, 11.615746], # 26
[0.653940, -42.193790, -13.380835], # 27
[0.804809, -30.993721, -21.150853], # 28
[0.992204, -19.944596, -29.284036], # 29
[1.226783, -8.414541, -36.948060], # 00
[-14.772472, 2.598255, -20.132003], # 01
[-7.180239, 4.751589, -23.536684], # 02
[0.555920, 6.562900, -25.944448], # 03
[8.272499, 4.661005, -23.695741], # 04
[15.214351, 2.643046, -20.858157], # 05
[-46.047290, -37.471411, 7.037989], # 06
[-37.674688, -42.730510, 3.021217], # 07
[-27.883856, -42.711517, 1.353629], # 08
[-19.648268, -36.754742, -0.111088], # 09
[-28.272965, -35.134493, -0.147273], # 10
[-38.082418, -34.919043, 1.476612], # 11
[19.265868, -37.032306, -0.665746], # 12
[27.894191, -43.342445, 0.247660], # 13
[37.437529, -43.110822, 1.696435], # 14
[45.170805, -38.086515, 4.894163], # 15
[38.196454, -35.532024, 0.282961], # 16
[28.764989, -35.484289, -1.172675], # 17
[-28.916267, 28.612716, -2.240310], # 18
[-17.533194, 22.172187, -15.934335], # 19
[-6.684590, 19.029051, -22.611355], # 20
[0.381001, 20.721118, -23.748437], # 21
[8.375443, 19.035460, -22.721995], # 22
[18.876618, 22.394109, -15.610679], # 23
[28.794412, 28.079924, -3.217393], # 24
[19.057574, 36.298248, -14.987997], # 25
[8.956375, 39.634575, -22.554245], # 26
[0.381549, 40.395647, -23.591626], # 27
[-7.428895, 39.836405, -22.406106], # 28
[-18.160634, 36.677899, -15.121907], # 29
[-24.377490, 28.677771, -4.785684], # 30
[-6.897633, 25.475976, -20.893742], # 31
[0.340663, 26.014269, -22.220479], # 32
[8.444722, 25.326198, -21.025520], # 33
[24.474473, 28.323008, -5.712776], # 34
[8.449166, 30.596216, -20.671489], # 35
[0.205322, 31.408738, -21.903670], # 36
[-7.198266, 30.844876, -20.328022] # 37
], dtype=np.float32)
FaceType_to_padding_remove_align = {
FaceType.HALF: (0.0, False),
FaceType.MID_FULL: (0.0675, False),
FaceType.FULL: (0.2109375, False),
FaceType.FULL_NO_ALIGN: (0.2109375, True),
FaceType.WHOLE_FACE: (0.40, False),
FaceType.HEAD: (0.70, False),
FaceType.HEAD_NO_ALIGN: (0.70, True),
}
def convert_98_to_68(lmrks):
# jaw
result = [lmrks[0]]
for i in range(2, 16, 2):
result += [(lmrks[i] + (lmrks[i - 1] + lmrks[i + 1]) / 2) / 2]
result += [lmrks[16]]
for i in range(18, 32, 2):
result += [(lmrks[i] + (lmrks[i - 1] + lmrks[i + 1]) / 2) / 2]
result += [lmrks[32]]
# eyebrows averaging
result += [lmrks[33],
(lmrks[34] + lmrks[41]) / 2,
(lmrks[35] + lmrks[40]) / 2,
(lmrks[36] + lmrks[39]) / 2,
(lmrks[37] + lmrks[38]) / 2,
]
result += [(lmrks[42] + lmrks[50]) / 2,
(lmrks[43] + lmrks[49]) / 2,
(lmrks[44] + lmrks[48]) / 2,
(lmrks[45] + lmrks[47]) / 2,
lmrks[46]
]
# nose
result += list(lmrks[51:60])
# left eye (from our view)
result += [lmrks[60],
lmrks[61],
lmrks[63],
lmrks[64],
lmrks[65],
lmrks[67]]
# right eye
result += [lmrks[68],
lmrks[69],
lmrks[71],
lmrks[72],
lmrks[73],
lmrks[75]]
# mouth
result += list(lmrks[76:96])
return np.concatenate(result).reshape((68, 2))
def transform_points(points, mat, invert=False):
if invert:
mat = cv2.invertAffineTransform(mat)
points = np.expand_dims(points, axis=1)
points = cv2.transform(points, mat, points.shape)
points = np.squeeze(points)
return points
def get_transform_mat(image_landmarks, output_size, face_type, scale=1.0):
if not isinstance(image_landmarks, np.ndarray):
image_landmarks = np.array(image_landmarks)
# estimate landmarks transform from global space to local aligned space with bounds [0..1]
mat = umeyama(np.concatenate([image_landmarks[17:49], image_landmarks[54:55]]), landmarks_2D_new, True)[0:2]
# get corner points in global space
g_p = transform_points(np.float32([(0, 0), (1, 0), (1, 1), (0, 1), (0.5, 0.5)]), mat, True)
g_c = g_p[4]
# calc diagonal vectors between corners in global space
tb_diag_vec = (g_p[2] - g_p[0]).astype(np.float32)
tb_diag_vec /= npla.norm(tb_diag_vec)
bt_diag_vec = (g_p[1] - g_p[3]).astype(np.float32)
bt_diag_vec /= npla.norm(bt_diag_vec)
# calc modifier of diagonal vectors for scale and padding value
# print(face_type)
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
mod = (1.0 / scale) * (npla.norm(g_p[0] - g_p[2]) * (padding * np.sqrt(2.0) + 0.5))
if face_type == FaceType.WHOLE_FACE:
# adjust vertical offset for WHOLE_FACE, 7% below in order to cover more forehead
vec = (g_p[0] - g_p[3]).astype(np.float32)
vec_len = npla.norm(vec)
vec /= vec_len
g_c += vec * vec_len * 0.07
# calc 3 points in global space to estimate 2d affine transform
if not remove_align:
l_t = np.array([g_c - tb_diag_vec * mod,
g_c + bt_diag_vec * mod,
g_c + tb_diag_vec * mod])
else:
# remove_align - face will be centered in the frame but not aligned
l_t = np.array([g_c - tb_diag_vec * mod,
g_c + bt_diag_vec * mod,
g_c + tb_diag_vec * mod,
g_c - bt_diag_vec * mod,
])
# get area of face square in global space
area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1])
# calc side of square
side = np.float32(math.sqrt(area) / 2)
# calc 3 points with unrotated square
l_t = np.array([g_c + [-side, -side],
g_c + [side, -side],
g_c + [side, side]])
# calc affine transform from 3 global space points to 3 local space points size of 'output_size'
pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size)))
mat = cv2.getAffineTransform(l_t, pts2)
return mat
def get_rect_from_landmarks(image_landmarks):
mat = get_transform_mat(image_landmarks, 256, FaceType.FULL_NO_ALIGN)
g_p = transform_points(np.float32([(0, 0), (255, 255)]), mat, True)
(l, t, r, b) = g_p[0][0], g_p[0][1], g_p[1][0], g_p[1][1]
return (l, t, r, b)
def get_transform_mat_all(image_landmarks,uni_landmarks,output_size,scale=1,gcx=-0.02,gcy=0.15,face_type=FaceType.WHOLE_FACE):
if not isinstance(image_landmarks, np.ndarray):
image_landmarks = np.array (image_landmarks)
# estimate landmarks transform from global space to local aligned space with bounds [0..1]
mat = umeyama(image_landmarks, uni_landmarks, True)[0:2]
# get corner points in global space
g_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5) ]) , mat, True)
g_c = g_p[4]
# calc diagonal vectors between corners in global space
tb_diag_vec = (g_p[2] - g_p[0]).astype(np.float32)
tb_diag_vec /= npla.norm(tb_diag_vec)
bt_diag_vec = (g_p[1] - g_p[3]).astype(np.float32)
bt_diag_vec /= npla.norm(bt_diag_vec)
# calc modifier of diagonal vectors for scale and padding value
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
mod = (1.0 / scale) * (npla.norm(g_p[0] - g_p[2]) * (padding * np.sqrt(2.0) + 0.5))
vec = (g_p[0]-g_p[3]).astype(np.float32)
vec_len = npla.norm(vec)
vec /= vec_len
g_c += vec*vec_len*[gcx,gcy]
# calc 3 points in global space to estimate 2d affine transform
if not remove_align:
l_t = np.array([g_c - tb_diag_vec * mod,
g_c + bt_diag_vec * mod,
g_c + tb_diag_vec * mod])
else:
# remove_align - face will be centered in the frame but not aligned
l_t = np.array([g_c - tb_diag_vec * mod,
g_c + bt_diag_vec * mod,
g_c + tb_diag_vec * mod,
g_c - bt_diag_vec * mod,
])
# get area of face square in global space
area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1])
# calc side of square
side = np.float32(math.sqrt(area) / 2)
# calc 3 points with unrotated square
l_t = np.array([g_c + [-side, -side],
g_c + [side, -side],
g_c + [side, side]])
# calc affine transform from 3 global space points to 3 local space points size of 'output_size'
pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size)))
mat = cv2.getAffineTransform(l_t, pts2)
return mat
def expand_eyebrows(lmrks, eyebrows_expand_mod=1.0):
if len(lmrks) != 68:
raise Exception('works only with 68 landmarks')
lmrks = np.array(lmrks.copy(), dtype=np.int)
# #nose
ml_pnt = (lmrks[36] + lmrks[0]) // 2
mr_pnt = (lmrks[16] + lmrks[45]) // 2
# mid points between the mid points and eye
ql_pnt = (lmrks[36] + ml_pnt) // 2
qr_pnt = (lmrks[45] + mr_pnt) // 2
# Top of the eye arrays
bot_l = np.array((ql_pnt, lmrks[36], lmrks[37], lmrks[38], lmrks[39]))
bot_r = np.array((lmrks[42], lmrks[43], lmrks[44], lmrks[45], qr_pnt))
# Eyebrow arrays
top_l = lmrks[17:22]
top_r = lmrks[22:27]
# Adjust eyebrow arrays
lmrks[17:22] = top_l + eyebrows_expand_mod * 0.5 * (top_l - bot_l)
lmrks[22:27] = top_r + eyebrows_expand_mod * 0.5 * (top_r - bot_r)
return lmrks
def get_image_hull_mask(image_shape, image_landmarks, eyebrows_expand_mod=1.0):
hull_mask = np.zeros(image_shape[0:2] + (1,), dtype=np.float32)
lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod)
r_jaw = (lmrks[0:9], lmrks[17:18])
l_jaw = (lmrks[8:17], lmrks[26:27])
r_cheek = (lmrks[17:20], lmrks[8:9])
l_cheek = (lmrks[24:27], lmrks[8:9])
nose_ridge = (lmrks[19:25], lmrks[8:9],)
r_eye = (lmrks[17:22], lmrks[27:28], lmrks[31:36], lmrks[8:9])
l_eye = (lmrks[22:27], lmrks[27:28], lmrks[31:36], lmrks[8:9])
nose = (lmrks[27:31], lmrks[31:36])
parts = [r_jaw, l_jaw, r_cheek, l_cheek, nose_ridge, r_eye, l_eye, nose]
for item in parts:
merged = np.concatenate(item)
cv2.fillConvexPoly(hull_mask, cv2.convexHull(merged), (1,))
return hull_mask
def get_image_eye_mask(image_shape, image_landmarks):
if len(image_landmarks) != 68:
raise Exception('get_image_eye_mask works only with 68 landmarks')
h, w, c = image_shape
hull_mask = np.zeros((h, w, 1), dtype=np.float32)
image_landmarks = image_landmarks.astype(np.int)
cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[36:42]), (1,))
cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[42:48]), (1,))
dilate = h // 32
hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate)), iterations=1)
blur = h // 16
blur = blur + (1 - blur % 2)
hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur), 0)
hull_mask = hull_mask[..., None]
return hull_mask
def get_image_mouth_mask(image_shape, image_landmarks):
if len(image_landmarks) != 68:
raise Exception('get_image_eye_mask works only with 68 landmarks')
h, w, c = image_shape
hull_mask = np.zeros((h, w, 1), dtype=np.float32)
image_landmarks = image_landmarks.astype(np.int)
cv2.fillConvexPoly(hull_mask, cv2.convexHull(image_landmarks[60:]), (1,))
dilate = h // 32
hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dilate, dilate)), iterations=1)
blur = h // 16
blur = blur + (1 - blur % 2)
hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur), 0)
hull_mask = hull_mask[..., None]
return hull_mask
def alpha_to_color(img_alpha, color):
if len(img_alpha.shape) == 2:
img_alpha = img_alpha[..., None]
h, w, c = img_alpha.shape
result = np.zeros((h, w, len(color)), dtype=np.float32)
result[:, :] = color
return result * img_alpha
def get_cmask(image_shape, lmrks, eyebrows_expand_mod=1.0):
h, w, c = image_shape
hull = get_image_hull_mask(image_shape, lmrks, eyebrows_expand_mod)
result = np.zeros((h, w, 3), dtype=np.float32)
def process(w, h, data):
d = {}
cur_lc = 0
all_lines = []
for s, pts_loop_ar in data:
lines = []
for pts, loop in pts_loop_ar:
pts_len = len(pts)
lines.append([[pts[i], pts[(i + 1) % pts_len]] for i in range(pts_len - (0 if loop else 1))])
lines = np.concatenate(lines)
lc = lines.shape[0]
all_lines.append(lines)
d[s] = cur_lc, cur_lc + lc
cur_lc += lc
all_lines = np.concatenate(all_lines, 0)
# calculate signed distance for all points and lines
line_count = all_lines.shape[0]
pts_count = w * h
all_lines = np.repeat(all_lines[None, ...], pts_count, axis=0).reshape((pts_count * line_count, 2, 2))
pts = np.empty((h, w, line_count, 2), dtype=np.float32)
pts[..., 1] = np.arange(h)[:, None, None]
pts[..., 0] = np.arange(w)[:, None]
pts = pts.reshape((h * w * line_count, -1))
a = all_lines[:, 0, :]
b = all_lines[:, 1, :]
pa = pts - a
ba = b - a
ph = np.clip(np.einsum('ij,ij->i', pa, ba) / np.einsum('ij,ij->i', ba, ba), 0, 1)
dists = npla.norm(pa - ba * ph[..., None], axis=1).reshape((h, w, line_count))
def get_dists(name, thickness=0):
s, e = d[name]
result = dists[..., s:e]
if thickness != 0:
result = np.abs(result) - thickness
return np.min(result, axis=-1)
return get_dists
l_eye = lmrks[42:48]
r_eye = lmrks[36:42]
l_brow = lmrks[22:27]
r_brow = lmrks[17:22]
mouth = lmrks[48:60]
up_nose = np.concatenate((lmrks[27:31], lmrks[33:34]))
down_nose = lmrks[31:36]
nose = np.concatenate((up_nose, down_nose))
gdf = process(w, h,
(
('eyes', ((l_eye, True), (r_eye, True))),
('brows', ((l_brow, False), (r_brow, False))),
('up_nose', ((up_nose, False),)),
('down_nose', ((down_nose, False),)),
('mouth', ((mouth, True),)),
)
)
eyes_fall_dist = w // 32
eyes_thickness = max(w // 64, 1)
brows_fall_dist = w // 32
brows_thickness = max(w // 256, 1)
nose_fall_dist = w / 12
nose_thickness = max(w // 96, 1)
mouth_fall_dist = w // 32
mouth_thickness = max(w // 64, 1)
eyes_mask = gdf('eyes', eyes_thickness)
eyes_mask = 1 - np.clip(eyes_mask / eyes_fall_dist, 0, 1)
# eyes_mask = np.clip ( 1- ( np.sqrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1)
# eyes_mask = np.clip ( 1- ( np.cbrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1)
brows_mask = gdf('brows', brows_thickness)
brows_mask = 1 - np.clip(brows_mask / brows_fall_dist, 0, 1)
# brows_mask = np.clip ( 1- ( np.sqrt( np.maximum(brows_mask,0) ) / brows_fall_dist ), 0, 1)
mouth_mask = gdf('mouth', mouth_thickness)
mouth_mask = 1 - np.clip(mouth_mask / mouth_fall_dist, 0, 1)
# mouth_mask = np.clip ( 1- ( np.sqrt( np.maximum(mouth_mask,0) ) / mouth_fall_dist ), 0, 1)
def blend(a, b, k):
x = np.clip(0.5 + 0.5 * (b - a) / k, 0.0, 1.0)
return (a - b) * x + b - k * x * (1.0 - x)
# nose_mask = (a-b)*x+b - k*x*(1.0-x)
# nose_mask = np.minimum (up_nose_mask , down_nose_mask )
# nose_mask = 1-np.clip( nose_mask / nose_fall_dist, 0, 1)
nose_mask = blend(gdf('up_nose', nose_thickness), gdf('down_nose', nose_thickness), nose_thickness * 3)
nose_mask = 1 - np.clip(nose_mask / nose_fall_dist, 0, 1)
up_nose_mask = gdf('up_nose', nose_thickness)
up_nose_mask = 1 - np.clip(up_nose_mask / nose_fall_dist, 0, 1)
# up_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(up_nose_mask,0) ) / nose_fall_dist ), 0, 1)
down_nose_mask = gdf('down_nose', nose_thickness)
down_nose_mask = 1 - np.clip(down_nose_mask / nose_fall_dist, 0, 1)
# down_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(down_nose_mask,0) ) / nose_fall_dist ), 0, 1)
# nose_mask = np.clip( up_nose_mask + down_nose_mask, 0, 1 )
# nose_mask /= np.max(nose_mask)
# nose_mask = np.maximum (up_nose_mask , down_nose_mask )
# nose_mask = down_nose_mask
# nose_mask = np.zeros_like(nose_mask)
eyes_mask = eyes_mask * (1 - mouth_mask)
nose_mask = nose_mask * (1 - eyes_mask)
hull_mask = hull[..., 0].copy()
hull_mask = hull_mask * (1 - eyes_mask) * (1 - brows_mask) * (1 - nose_mask) * (1 - mouth_mask)
# eyes_mask = eyes_mask * (1-nose_mask)
mouth_mask = mouth_mask * (1 - nose_mask)
brows_mask = brows_mask * (1 - nose_mask) * (1 - eyes_mask)
hull_mask = alpha_to_color(hull_mask, (0, 1, 0))
eyes_mask = alpha_to_color(eyes_mask, (1, 0, 0))
brows_mask = alpha_to_color(brows_mask, (0, 0, 1))
nose_mask = alpha_to_color(nose_mask, (0, 1, 1))
mouth_mask = alpha_to_color(mouth_mask, (0, 0, 1))
# nose_mask = np.maximum( up_nose_mask, down_nose_mask )
result = hull_mask + mouth_mask + nose_mask + brows_mask + eyes_mask
result *= hull
# result = np.clip (result, 0, 1)
return result
def blur_image_hull_mask(hull_mask):
maxregion = np.argwhere(hull_mask == 1.0)
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx;
leny = maxy - miny;
masky = int(minx + (lenx // 2))
maskx = int(miny + (leny // 2))
lowest_len = min(lenx, leny)
ero = int(lowest_len * 0.085)
blur = int(lowest_len * 0.10)
hull_mask = cv2.erode(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1)
hull_mask = cv2.blur(hull_mask, (blur, blur))
hull_mask = np.expand_dims(hull_mask, -1)
return hull_mask
mirror_idxs = [
[0, 16],
[1, 15],
[2, 14],
[3, 13],
[4, 12],
[5, 11],
[6, 10],
[7, 9],
[17, 26],
[18, 25],
[19, 24],
[20, 23],
[21, 22],
[36, 45],
[37, 44],
[38, 43],
[39, 42],
[40, 47],
[41, 46],
[31, 35],
[32, 34],
[50, 52],
[49, 53],
[48, 54],
[59, 55],
[58, 56],
[67, 65],
[60, 64],
[61, 63]]
def mirror_landmarks(landmarks, val):
result = landmarks.copy()
for idx in mirror_idxs:
result[idx] = result[idx[::-1]]
result[:, 0] = val - result[:, 0] - 1
return result
def get_face_struct_mask(image_shape, image_landmarks, eyebrows_expand_mod=1.0, color=(1,)):
mask = np.zeros(image_shape[0:2] + (len(color),), dtype=np.float32)
lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod)
draw_landmarks(mask, image_landmarks, color=color, draw_circles=False, thickness=2)
return mask
def draw_landmarks(image, image_landmarks, color=(0, 255, 0), draw_circles=True, thickness=1, transparent_mask=False):
if len(image_landmarks) != 68:
raise Exception('get_image_eye_mask works only with 68 landmarks')
int_lmrks = np.array(image_landmarks, dtype=np.int)
jaw = int_lmrks[slice(*landmarks_68_pt["jaw"])]
right_eyebrow = int_lmrks[slice(*landmarks_68_pt["right_eyebrow"])]
left_eyebrow = int_lmrks[slice(*landmarks_68_pt["left_eyebrow"])]
mouth = int_lmrks[slice(*landmarks_68_pt["mouth"])]
right_eye = int_lmrks[slice(*landmarks_68_pt["right_eye"])]
left_eye = int_lmrks[slice(*landmarks_68_pt["left_eye"])]
nose = int_lmrks[slice(*landmarks_68_pt["nose"])]
# open shapes
cv2.polylines(image,
tuple(np.array([v]) for v in (right_eyebrow, jaw, left_eyebrow, np.concatenate((nose, [nose[-6]])))),
False, color, thickness=thickness, lineType=cv2.LINE_AA)
# closed shapes
cv2.polylines(image, tuple(np.array([v]) for v in (right_eye, left_eye, mouth)),
True, color, thickness=thickness, lineType=cv2.LINE_AA)
if draw_circles:
# the rest of the cicles
for x, y in np.concatenate((right_eyebrow, left_eyebrow, mouth, right_eye, left_eye, nose), axis=0):
cv2.circle(image, (x, y), 1, color, 1, lineType=cv2.LINE_AA)
# jaw big circles
for x, y in jaw:
cv2.circle(image, (x, y), 2, color, lineType=cv2.LINE_AA)
if transparent_mask:
mask = get_image_hull_mask(image.shape, image_landmarks)
image[...] = (image * (1 - mask) + image * mask / 2)[...]
def draw_rect_landmarks(image, rect, image_landmarks, face_type, face_size=256, transparent_mask=False,
landmarks_color=(0, 255, 0)):
draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask)
imagelib.draw_rect(image, rect, (255, 0, 0), 2)
image_to_face_mat = get_transform_mat(image_landmarks, face_size, face_type)
points = transform_points([(0, 0), (0, face_size - 1), (face_size - 1, face_size - 1), (face_size - 1, 0)],
image_to_face_mat, True)
imagelib.draw_polygon(image, points, (0, 0, 255), 2)
points = transform_points(
[(int(face_size * 0.05), 0), (int(face_size * 0.1), int(face_size * 0.1)), (0, int(face_size * 0.1))],
image_to_face_mat, True)
imagelib.draw_polygon(image, points, (0, 0, 255), 2)
def calc_face_pitch(landmarks):
if not isinstance(landmarks, np.ndarray):
landmarks = np.array(landmarks)
t = ((landmarks[6][1] - landmarks[8][1]) + (landmarks[10][1] - landmarks[8][1])) / 2.0
b = landmarks[8][1]
return float(b - t)
def estimate_averaged_yaw(landmarks):
# Works much better than solvePnP if landmarks from "3DFAN"
if not isinstance(landmarks, np.ndarray):
landmarks = np.array(landmarks)
l = ((landmarks[27][0] - landmarks[0][0]) + (landmarks[28][0] - landmarks[1][0]) + (
landmarks[29][0] - landmarks[2][0])) / 3.0
r = ((landmarks[16][0] - landmarks[27][0]) + (landmarks[15][0] - landmarks[28][0]) + (
landmarks[14][0] - landmarks[29][0])) / 3.0
return float(r - l)
def estimate_pitch_yaw_roll(aligned_landmarks, size=256):
"""
returns pitch,yaw,roll [-pi/2...+pi/2]
"""
shape = (size, size)
focal_length = shape[1]
camera_center = (shape[1] / 2, shape[0] / 2)
camera_matrix = np.array(
[[focal_length, 0, camera_center[0]],
[0, focal_length, camera_center[1]],
[0, 0, 1]], dtype=np.float32)
(_, rotation_vector, _) = cv2.solvePnP(
np.concatenate((landmarks_68_3D[:27], landmarks_68_3D[30:36]), axis=0),
np.concatenate((aligned_landmarks[:27], aligned_landmarks[30:36]), axis=0).astype(np.float32),
camera_matrix,
np.zeros((4, 1)))
pitch, yaw, roll = mathlib.rotationMatrixToEulerAngles(cv2.Rodrigues(rotation_vector)[0])
half_pi = math.pi / 2.0
pitch = np.clip(pitch, -half_pi, half_pi)
yaw = np.clip(yaw, -half_pi, half_pi)
roll = np.clip(roll, -half_pi, half_pi)
return -pitch, yaw, roll
# if remove_align:
# bbox = transform_points ( [ (0,0), (0,output_size), (output_size, output_size), (output_size,0) ], mat, True)
# #import code
# #code.interact(local=dict(globals(), **locals()))
# area = mathlib.polygon_area(bbox[:,0], bbox[:,1] )
# side = math.sqrt(area) / 2
# center = transform_points ( [(output_size/2,output_size/2)], mat, True)
# pts1 = np.float32(( center+[-side,-side], center+[side,-side], center+[side,-side] ))
# pts2 = np.float32([[0,0],[output_size,0],[0,output_size]])
# mat = cv2.getAffineTransform(pts1,pts2)
# if full_face_align_top and (face_type == FaceType.FULL or face_type == FaceType.FULL_NO_ALIGN):
# #lmrks2 = expand_eyebrows(image_landmarks)
# #lmrks2_ = transform_points( [ lmrks2[19], lmrks2[24] ], mat, False )
# #y_diff = np.float32( (0,np.min(lmrks2_[:,1])) )
# #y_diff = transform_points( [ np.float32( (0,0) ), y_diff], mat, True)
# #y_diff = y_diff[1]-y_diff[0]
#
# x_diff = np.float32((0,0))
#
# lmrks2_ = transform_points( [ image_landmarks[0], image_landmarks[16] ], mat, False )
# if lmrks2_[0,0] < 0:
# x_diff = lmrks2_[0,0]
# x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True)
# x_diff = x_diff[1]-x_diff[0]
# elif lmrks2_[1,0] >= output_size:
# x_diff = lmrks2_[1,0]-(output_size-1)
# x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True)
# x_diff = x_diff[1]-x_diff[0]
#
# mat = cv2.getAffineTransform( l_t+y_diff+x_diff ,pts2)
"""
def get_averaged_transform_mat (img_landmarks,
img_landmarks_prev,
img_landmarks_next,
average_frame_count,
average_center_frame_count,
output_size, face_type, scale=1.0):
l_c_list = []
tb_diag_vec_list = []
bt_diag_vec_list = []
mod_list = []
count = max(average_frame_count,average_center_frame_count)
for i in range ( -count, count+1, 1 ):
if i < 0:
lmrks = img_landmarks_prev[i] if -i < len(img_landmarks_prev) else None
elif i > 0:
lmrks = img_landmarks_next[i] if i < len(img_landmarks_next) else None
else:
lmrks = img_landmarks
if lmrks is None:
continue
l_c, tb_diag_vec, bt_diag_vec, mod = get_transform_mat_data (lmrks, face_type, scale=scale)
if i >= -average_frame_count and i <= average_frame_count:
tb_diag_vec_list.append(tb_diag_vec)
bt_diag_vec_list.append(bt_diag_vec)
mod_list.append(mod)
if i >= -average_center_frame_count and i <= average_center_frame_count:
l_c_list.append(l_c)
tb_diag_vec = np.mean( np.array(tb_diag_vec_list), axis=0 )
bt_diag_vec = np.mean( np.array(bt_diag_vec_list), axis=0 )
mod = np.mean( np.array(mod_list), axis=0 )
l_c = np.mean( np.array(l_c_list), axis=0 )
return get_transform_mat_by_data (l_c, tb_diag_vec, bt_diag_vec, mod, output_size, face_type)
def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
if not isinstance(image_landmarks, np.ndarray):
image_landmarks = np.array (image_landmarks)
# get face padding value for FaceType
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
# estimate landmarks transform from global space to local aligned space with bounds [0..1]
mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2]
# get corner points in global space
l_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5)]) , mat, True)
l_c = l_p[4]
# calc diagonal vectors between corners in global space
tb_diag_vec = (l_p[2]-l_p[0]).astype(np.float32)
tb_diag_vec /= npla.norm(tb_diag_vec)
bt_diag_vec = (l_p[1]-l_p[3]).astype(np.float32)
bt_diag_vec /= npla.norm(bt_diag_vec)
# calc modifier of diagonal vectors for scale and padding value
mod = (1.0 / scale)* ( npla.norm(l_p[0]-l_p[2])*(padding*np.sqrt(2.0) + 0.5) )
# calc 3 points in global space to estimate 2d affine transform
if not remove_align:
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
np.round( l_c + bt_diag_vec*mod ),
np.round( l_c + tb_diag_vec*mod ) ] )
else:
# remove_align - face will be centered in the frame but not aligned
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
np.round( l_c + bt_diag_vec*mod ),
np.round( l_c + tb_diag_vec*mod ),
np.round( l_c - bt_diag_vec*mod ),
] )
# get area of face square in global space
area = mathlib.polygon_area(l_t[:,0], l_t[:,1] )
# calc side of square
side = np.float32(math.sqrt(area) / 2)
# calc 3 points with unrotated square
l_t = np.array( [ np.round( l_c + [-side,-side] ),
np.round( l_c + [ side,-side] ),
np.round( l_c + [ side, side] ) ] )
# calc affine transform from 3 global space points to 3 local space points size of 'output_size'
pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) ))
mat = cv2.getAffineTransform(l_t,pts2)
return mat
"""