import subprocess from PIL import Image,ImageOps,ImageDraw,ImageFilter import json import os import time import io from mp_utils import get_pixel_cordinate_list,extract_landmark,get_pixel_cordinate,get_normalized_xyz from glibvision.draw_utils import points_to_box,box_to_xy,plus_point,calculate_distance import numpy as np from glibvision.pil_utils import fill_points,create_color_image,draw_box import glibvision.pil_utils from gradio_utils import save_image,save_buffer,clear_old_files ,read_file import math import mp_triangles from glibvision.cv2_utils import create_color_image as cv2_create_color_image,copy_image,pil_to_bgr_image import cv2 #TODO move to CV2 # i'm not sure this is fast def apply_affine_transformation_to_triangle_add(src_tri, dst_tri, src_img, dst_img): src_tri_np = np.float32(src_tri) dst_tri_np = np.float32(dst_tri) h_dst, w_dst = dst_img.shape[:2] M = cv2.getAffineTransform(src_tri_np, dst_tri_np) dst_mask = np.zeros((h_dst, w_dst), dtype=np.uint8) cv2.fillPoly(dst_mask, [np.int32(dst_tri)], 255) transformed = cv2.warpAffine(src_img, M, (w_dst, h_dst)) transformed = transformed * (dst_mask[:, :, np.newaxis] / 255).astype(np.uint8) dst_background = dst_img * (1 - (dst_mask[:, :, np.newaxis] / 255)).astype(np.uint8) dst_img = transformed + dst_background return dst_img def apply_affine_transformation_to_triangle_add(src_tri, dst_tri, src_img, dst_img): src_tri_np = np.float32(src_tri) dst_tri_np = np.float32(dst_tri) assert src_tri_np.shape == (3, 2), f"src_tri_np の形状が不正 {src_tri_np.shape}" assert dst_tri_np.shape == (3, 2), f"dst_tri_np の形状が不正 {dst_tri_np.shape}" # 透視変換行列の計算 M = cv2.getAffineTransform(src_tri_np, dst_tri_np) # 画像のサイズ h_src, w_src = src_img.shape[:2] h_dst, w_dst = dst_img.shape[:2] # 元画像から三角形領域を切り抜くマスク生成 #src_mask = np.zeros((h_src, w_src), dtype=np.uint8) #cv2.fillPoly(src_mask, [np.int32(src_tri)], 255) # Not 元画像の三角形領域のみをマスクで抽出 src_triangle = src_img #cv2.bitwise_and(src_img, src_img, mask=src_mask) # 変換行列を使って元画像の三角形領域を目標画像のサイズへ変換 transformed = cv2.warpAffine(src_triangle, M, (w_dst, h_dst)) #print(f"dst_img={dst_img.shape}") #print(f"transformed={transformed.shape}") # 変換後のマスクの生成 dst_mask = np.zeros((h_dst, w_dst), dtype=np.uint8) cv2.fillPoly(dst_mask, [np.int32(dst_tri)], 255) transformed = cv2.bitwise_and(transformed, transformed, mask=dst_mask) # 目標画像のマスク領域をクリアするためにデストのインバートマスクを作成 dst_mask_inv = cv2.bitwise_not(dst_mask) # 目標画像のマスク部分をクリア dst_background = cv2.bitwise_and(dst_img, dst_img, mask=dst_mask_inv) # 変換された元画像の三角形部分と目標画像の背景部分を合成 dst_img = cv2.add(dst_background, transformed) return dst_img # TODO move PIL def process_create_webp(images,duration=100, loop=0,quality=85): frames = [] for image_file in images: frames.append(image_file) output_buffer = io.BytesIO() frames[0].save(output_buffer, save_all=True, append_images=frames[1:], duration=duration, loop=loop, format='WebP', quality=quality ) return output_buffer.getvalue() # TODO move numpy def rotate_point_euler(point, angles,order="xyz"): """ オイラー角を使って3Dポイントを回転させる関数 Args: point: 回転させる3Dポイント (x, y, z) angles: 各軸周りの回転角度 (rx, ry, rz) [ラジアン] Returns: 回転後の3Dポイント (x', y', z') """ rx, ry, rz = angles point = np.array(point) # X軸周りの回転 Rx = np.array([ [1, 0, 0], [0, np.cos(rx), -np.sin(rx)], [0, np.sin(rx), np.cos(rx)] ]) # Y軸周りの回転 Ry = np.array([ [np.cos(ry), 0, np.sin(ry)], [0, 1, 0], [-np.sin(ry), 0, np.cos(ry)] ]) # Z軸周りの回転 Rz = np.array([ [np.cos(rz), -np.sin(rz), 0], [np.sin(rz), np.cos(rz), 0], [0, 0, 1] ]) # 回転行列の合成 (Z軸 -> Y軸 -> X軸 の順で回転) order = order.lower() if order == "xyz": R = Rx @ Ry @ Rz elif order == "xzy": R = Rx @ Rz @ Ry elif order == "yxz": R = Ry @ Rx @ Rz elif order == "yzx": R = Ry @ Rz @ Rx elif order == "zxy": R = Rz @ Rx @ Ry else: R = Rz @ Ry @ Rx # 回転後のポイントを計算 rotated_point = R @ point return rotated_point def process_face_mesh_spinning(image,draw_type,center_scaleup,animation_direction,z_multiply=0.8,inner_eyes=False,inner_mouth=False): animation = True offset_x = 0 offset_y = 0 # use when center_scaleup is True,scale is 0.45(half-size:0.5-margin/ nosetip-to-top or nosetip-to-bottom scale_up = 1.0 face_landmarker_result = None if image == None:#app stop support none image,if mode still image,make problem # Box for no Image Case image_width = 512 image_height = 512 #image = create_color_image(image_width,image_height,(0,0,0)) points = [(-0.25,-0.25,0),(0.25,-0.25,0), (0.25,0.25,0),(-0.25,0.25,0) ] normalized_center_point = [0.5,0.5] else: image_width = image.width image_height = image.height mp_image,face_landmarker_result = extract_landmark(image,"face_landmarker.task",0,0,True) def rotate_image(): return None,face_landmarker_result,None #return rotate_image() # cordinate eyes # cordinate all landmark_points = [get_normalized_xyz(face_landmarker_result.face_landmarks,i) for i in range(0,468)] # do centering normalized_center_point = landmark_points[4] normalized_top_point = landmark_points[10] normalized_bottom_point = landmark_points[152] offset_x = normalized_center_point[0] offset_y = normalized_center_point[1] offset_z = normalized_center_point[2] #need aspect? points = [[point[0]-offset_x,point[1]-offset_y,point[2]*z_multiply] for point in landmark_points] # split xy-cordinate and z-depth def split_points_xy_z(points,width,height,center_x,center_y): xys = [] zs = [] for point in points: xys.append( [ point[0]*width*scale_up+center_x, point[1]*height*scale_up+center_y ] ) zs.append(point[2]) return xys,zs def draw_grid_in_center(draw,cx,cy,grid_size,grid_color,width=1,draw_horizontal=True,draw_vertical=True): w = image.width h = image.height x_minus_divide = cx//grid_size x_plus_divide = (w -cx)//grid_size y_minus_divide = cy//grid_size y_plus_divide = (h -cx)//grid_size for i in range(-x_minus_divide,x_plus_divide+1): draw.line([(cx+i*grid_size,0),(cx+i*grid_size,h)],fill=grid_color,width=width) for i in range(-y_minus_divide,y_plus_divide+1): draw.line([(0,cy+i*grid_size),(w,cy+i*grid_size)],fill=grid_color,width=width) def draw_grid(image,cx=512,cy=512,first_color=(255,0,0)): w = image.width h = image.height second_grid_size=100 second_color = (128,128,128) draw = ImageDraw.Draw(image) draw_grid_in_center(draw,cx,cy,20,(100,100,100)) draw_grid_in_center(draw,cx,cy,100,(192,192,192)) draw.line([(cx,0),(cx,image.height)],fill=first_color) draw.line([(0,cy),(image.width,cy)],fill=first_color) def create_triangle_image(points,width,height,center_x,center_y,line_color=(255,255,255),fill_color=None): cordinates,angled_depth = split_points_xy_z(points,width,height,center_x,center_y) img = create_color_image(width,height,(0,0,0)) draw = ImageDraw.Draw(img) triangles = mp_triangles.get_triangles_copy(True,inner_eyes,inner_eyes,inner_mouth) triangles.sort(key=lambda triangle: sum(angled_depth[index] for index in triangle) / len(triangle) ,reverse=True) for triangle in triangles: triangle_cordinates = [cordinates[index] for index in triangle] glibvision.pil_utils.image_draw_points(draw,triangle_cordinates,line_color,fill_color) return img def create_texture_image(image,origin_points,angled_points,width,height,center_x,center_y,line_color=(255,255,255),fill_color=None): cv2_image = pil_to_bgr_image(image) #print(f"shape={cv2_image.shape}") #cv2.imwrite("tmp.jpg",cv2_image) original_cordinates = [] cordinates,angled_depth = split_points_xy_z(angled_points,width,height,center_x,center_y) # original point need offset for point in origin_points: original_cordinates.append( [ (point[0]+offset_x)*width, (point[1]+offset_y)*height ] ) if cv2_image.shape[2]==3: cv2_bg_img = cv2_create_color_image(cv2_image,(0,0,0)) else: cv2_bg_img = cv2_create_color_image(cv2_image,(0,0,0,0)) triangles = mp_triangles.get_triangles_copy(True,inner_eyes,inner_eyes,inner_mouth) triangles.sort(key=lambda triangle: sum(angled_depth[index] for index in triangle) / len(triangle) ,reverse=True) for triangle in triangles: triangle_cordinates = [cordinates[index] for index in triangle] origin_triangle_cordinates = [original_cordinates[index] for index in triangle] cv2_bg_img=apply_affine_transformation_to_triangle_add(origin_triangle_cordinates,triangle_cordinates,cv2_image,cv2_bg_img) img= Image.fromarray(cv2.cvtColor(cv2_bg_img, cv2.COLOR_RGBA2BGRA)) return img def create_point_image(points,width,height,center_x,center_y): cordinates,_ = split_points_xy_z(points,width,height,center_x,center_y) img = create_color_image(width,height,(0,0,0)) glibvision.pil_utils.draw_points(img,cordinates,None,None,3,(255,0,0),3) return img def angled_points(points,angles,order="xyz"): angled_cordinates = [] for point in points: rotated_np_point = rotate_point_euler(point,angles,order) angled_cordinates.append( [ rotated_np_point[0], rotated_np_point[1],rotated_np_point[2] ] ) return angled_cordinates frames = [] #frames.append(create_point_image(points)) frame_duration=100 start_angle=0 end_angle=360 step_angle=10 if draw_type == "Image": start_angle=-90 end_angle=90 step_angle=30 if not animation: start_angle=0 end_angle=0 step_angle=360 if image == None: draw_type="Dot" if center_scaleup and image!=None: top_distance = calculate_distance(normalized_center_point,normalized_top_point) bottom_distance = calculate_distance(normalized_center_point,normalized_bottom_point) distance = top_distance if top_distance>bottom_distance else bottom_distance #small_size = image_width if image_widthimage_width: scale_up *= image_width/image_height #print(scale_up) face_center_x = int(0.5* image_width)#half face_center_y = int(0.5* image_height) else: scale_up = 1.0 face_center_x = int(normalized_center_point[0]* image_width) face_center_y = int(normalized_center_point[1]* image_height) rotated_points = None if animation: for i in range(start_angle,end_angle,step_angle): if animation_direction == "X": angles = [math.radians(i),0,0] elif animation_direction == "Y": angles = [0,math.radians(i),0] else: angles = [0,0,math.radians(i)] if draw_type == "Dot": frames.append(create_point_image(angled_points(points,angles),image_width,image_height,face_center_x,face_center_y)) elif draw_type == "Line": frames.append(create_triangle_image(angled_points(points,angles),image_width,image_height,face_center_x,face_center_y)) elif draw_type == "Line+Fill": frames.append(create_triangle_image(angled_points(points,angles),image_width,image_height,face_center_x,face_center_y,(128,128,128),(200,200,200))) elif draw_type == "Image": frame_duration=500 frames.append(create_texture_image(image,points,angled_points(points,angles),image_width,image_height,face_center_x,face_center_y)) webp = process_create_webp(frames,frame_duration) path = save_buffer(webp) return path,face_landmarker_result,rotated_points