From 71e5df796af0f8c5c50350ca35d783d070d995d9 Mon Sep 17 00:00:00 2001 From: Alan Date: Wed, 15 Apr 2020 19:56:06 +0300 Subject: [PATCH 1/2] Face swap first commit --- .gitignore | 2 + hw3/app.py | 254 ++++++++++++++++++++++++++++++++++++++++++ hw3/download_model.sh | 3 + 3 files changed, 259 insertions(+) create mode 100644 hw3/app.py create mode 100644 hw3/download_model.sh diff --git a/.gitignore b/.gitignore index f70cbc8..1eb699c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ __pycache__/ .ipynb_checkpoints/ .idea/ + +assignment_2/ diff --git a/hw3/app.py b/hw3/app.py new file mode 100644 index 0000000..84b01a9 --- /dev/null +++ b/hw3/app.py @@ -0,0 +1,254 @@ +import numpy as np +import cv2 +import os +from flask import Flask, request, redirect, url_for, send_from_directory +import dlib + +# python version 3.6.8 + +RESULT_IMG_NAME = 'result.png' +UPLOAD_FOLDER = 'uploads' +ALLOWED_EXTENSIONS = ['png', 'jpg', 'jpeg'] + +app = Flask(__name__) +app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER + + +# Apply affine transform calculated using srcTri and dstTri to src and +# output an image of size. +def apply_affine_transform(src, srcTri, dstTri, size): + # Given a pair of triangles, find the affine transform. + warpMat = cv2.getAffineTransform(np.float32(srcTri), np.float32(dstTri)) + + # Apply the Affine Transform just found to the src image + dst = cv2.warpAffine(src, warpMat, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_REFLECT_101) + + return dst + + +# Check if a point is inside a rectangle +def rect_contains(rect, point): + if point[0] < rect[0]: + return False + elif point[1] < rect[1]: + return False + elif point[0] > rect[0] + rect[2]: + return False + elif point[1] > rect[1] + rect[3]: + return False + return True + + +# calculate delanauy triangle +def calculate_delaunay_triangles(rect, points): + # create subdiv + subdiv = cv2.Subdiv2D(rect) + + # Insert points into subdiv + for p in points: + subdiv.insert(p) + + triangleList = subdiv.getTriangleList() + + delaunayTri = [] + + pt = [] + + for t in triangleList: + pt.append((t[0], t[1])) + pt.append((t[2], t[3])) + pt.append((t[4], t[5])) + + pt1 = (t[0], t[1]) + pt2 = (t[2], t[3]) + pt3 = (t[4], t[5]) + + if rect_contains(rect, pt1) and rect_contains(rect, pt2) and rect_contains(rect, pt3): + ind = [] + # Get face-points (from 68 face detector) by coordinates + for j in range(0, 3): + for k in range(0, len(points)): + if abs(pt[j][0] - points[k][0]) < 1.0 and abs(pt[j][1] - points[k][1]) < 1.0: + ind.append(k) + # Three points form a triangle. Triangle array corresponds to the file tri.txt in FaceMorph + if len(ind) == 3: + delaunayTri.append((ind[0], ind[1], ind[2])) + + pt = [] + + return delaunayTri + + +# Warps and alpha blends triangular regions from img1 and img2 to img +def warp_triangle(img1, img2, t1, t2): + # Find bounding rectangle for each triangle + r1 = cv2.boundingRect(np.float32([t1])) + r2 = cv2.boundingRect(np.float32([t2])) + + # Offset points by left top corner of the respective rectangles + t1Rect = [] + t2Rect = [] + t2RectInt = [] + + for i in range(0, 3): + t1Rect.append(((t1[i][0] - r1[0]), (t1[i][1] - r1[1]))) + t2Rect.append(((t2[i][0] - r2[0]), (t2[i][1] - r2[1]))) + t2RectInt.append(((t2[i][0] - r2[0]), (t2[i][1] - r2[1]))) + + # Get mask by filling triangle + mask = np.zeros((r2[3], r2[2], 3), dtype=np.float32) + cv2.fillConvexPoly(mask, np.int32(t2RectInt), (1.0, 1.0, 1.0), 16, 0); + + # Apply warpImage to small rectangular patches + img1Rect = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]] + # img2Rect = np.zeros((r2[3], r2[2]), dtype = img1Rect.dtype) + + size = (r2[2], r2[3]) + + img2Rect = apply_affine_transform(img1Rect, t1Rect, t2Rect, size) + + img2Rect = img2Rect * mask + + # Copy triangular region of the rectangular patch to the output image + img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] = img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] * ( + (1.0, 1.0, 1.0) - mask) + + img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] = img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] + img2Rect + + +def get_points(img): + points = [] + + detector = dlib.get_frontal_face_detector() + predictor = dlib.shape_predictor('model.dat') + + gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + + # only one face + face = detector(gray_img)[0] + landmarks = predictor(gray_img, face) + + # predictor yields 68 points + for point in range(0, 68): + x = landmarks.part(point).x + y = landmarks.part(point).y + points.append((x, y)) + + return points + + +def swap_faces(source1, source2): + # Read images + img1 = cv2.imread(source1) + img2 = cv2.imread(source2) + img1Warped = np.copy(img2) + + # Read array of corresponding points + points1 = get_points(img1) + points2 = get_points(img2) + + # Find convex hull + hull1 = [] + hull2 = [] + + hullIndex = cv2.convexHull(np.array(points2), returnPoints=False) + + for i in range(0, len(hullIndex)): + hull1.append(points1[int(hullIndex[i])]) + hull2.append(points2[int(hullIndex[i])]) + + # Find delanauy traingulation for convex hull points + sizeImg2 = img2.shape + rect = (0, 0, sizeImg2[1], sizeImg2[0]) + + dt = calculate_delaunay_triangles(rect, hull2) + + if len(dt) == 0: + quit() + + # Apply affine transformation to Delaunay triangles + for i in range(0, len(dt)): + t1 = [] + t2 = [] + + # get points for img1, img2 corresponding to the triangles + for j in range(0, 3): + t1.append(hull1[dt[i][j]]) + t2.append(hull2[dt[i][j]]) + + warp_triangle(img1, img1Warped, t1, t2) + + # Calculate Mask + hull8U = [] + for i in range(0, len(hull2)): + hull8U.append((hull2[i][0], hull2[i][1])) + + mask = np.zeros(img2.shape, dtype=img2.dtype) + + cv2.fillConvexPoly(mask, np.int32(hull8U), (255, 255, 255)) + + r = cv2.boundingRect(np.float32([hull2])) + + center = (r[0] + int(r[2] / 2), r[1] + int(r[3] / 2)) + + # Clone seamlessly. + output = cv2.seamlessClone(np.uint8(img1Warped), img2, mask, center, cv2.NORMAL_CLONE) + cv2.imwrite(os.path.join(app.config['UPLOAD_FOLDER'], RESULT_IMG_NAME), output) + + +# Нагло сдул у Анжелы всё, что ниже +def allowed_file(filename): + return '.' in filename and \ + filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS + + +@app.route('/', methods=['GET', 'POST']) +def upload(): + if request.method == 'POST': + img1 = request.files['img1'] + img2 = request.files['img2'] + if img1 and allowed_file(img1.filename) and img2 and allowed_file(img2.filename): + img1.save(os.path.join(app.config['UPLOAD_FOLDER'], img1.filename)) + img2.save(os.path.join(app.config['UPLOAD_FOLDER'], img2.filename)) + swap_faces(img1.filename, img2.filename) + return redirect(url_for('uploaded_file', filename=RESULT_IMG_NAME)) + + return ''' + + + + + + Upload new images + + +
+
+

+

+ +
+
+ + + ''' + + +@app.route('/uploads/') +def uploaded_file(filename): + return send_from_directory(app.config['UPLOAD_FOLDER'], filename) + + +if __name__ == '__main__': + + if not os.path.exists('model.dat'): + os.system('download_model.sh') + + if not os.path.exists(UPLOAD_FOLDER): + os.makedirs(UPLOAD_FOLDER) + + app.run() diff --git a/hw3/download_model.sh b/hw3/download_model.sh new file mode 100644 index 0000000..b304169 --- /dev/null +++ b/hw3/download_model.sh @@ -0,0 +1,3 @@ +wget http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2 -O model.dat.bz2 +7z x model.dat.bz2 +rm model.dat.bz2 \ No newline at end of file From 56ed09196ff2d2a575ac6011b9b3320ccb5e6cfd Mon Sep 17 00:00:00 2001 From: Alan Date: Mon, 16 Nov 2020 07:27:20 +0300 Subject: [PATCH 2/2] Codestyle fix --- hw3/app.py | 80 +++++++++++++++++++++++++----------------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/hw3/app.py b/hw3/app.py index 84b01a9..2111264 100644 --- a/hw3/app.py +++ b/hw3/app.py @@ -16,12 +16,12 @@ # Apply affine transform calculated using srcTri and dstTri to src and # output an image of size. -def apply_affine_transform(src, srcTri, dstTri, size): +def apply_affine_transform(src, src_tri, dst_tri, size): # Given a pair of triangles, find the affine transform. - warpMat = cv2.getAffineTransform(np.float32(srcTri), np.float32(dstTri)) + warp_mat = cv2.getAffineTransform(np.float32(src_tri), np.float32(dst_tri)) # Apply the Affine Transform just found to the src image - dst = cv2.warpAffine(src, warpMat, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, + dst = cv2.warpAffine(src, warp_mat, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101) return dst @@ -29,15 +29,11 @@ def apply_affine_transform(src, srcTri, dstTri, size): # Check if a point is inside a rectangle def rect_contains(rect, point): - if point[0] < rect[0]: + if not rect[0] < point[0] < rect[0] + rect[2] or\ + not rect[1] < point[1] < rect[1] + rect[3]: return False - elif point[1] < rect[1]: - return False - elif point[0] > rect[0] + rect[2]: - return False - elif point[1] > rect[1] + rect[3]: - return False - return True + else: + return True # calculate delanauy triangle @@ -49,21 +45,19 @@ def calculate_delaunay_triangles(rect, points): for p in points: subdiv.insert(p) - triangleList = subdiv.getTriangleList() + triangle_list = subdiv.getTriangleList() - delaunayTri = [] + delaunay_tri = [] pt = [] - for t in triangleList: - pt.append((t[0], t[1])) - pt.append((t[2], t[3])) - pt.append((t[4], t[5])) - + for t in triangle_list: pt1 = (t[0], t[1]) pt2 = (t[2], t[3]) pt3 = (t[4], t[5]) + pt += [pt1, pt2, pt3] + if rect_contains(rect, pt1) and rect_contains(rect, pt2) and rect_contains(rect, pt3): ind = [] # Get face-points (from 68 face detector) by coordinates @@ -73,11 +67,11 @@ def calculate_delaunay_triangles(rect, points): ind.append(k) # Three points form a triangle. Triangle array corresponds to the file tri.txt in FaceMorph if len(ind) == 3: - delaunayTri.append((ind[0], ind[1], ind[2])) + delaunay_tri.append((ind[0], ind[1], ind[2])) pt = [] - return delaunayTri + return delaunay_tri # Warps and alpha blends triangular regions from img1 and img2 to img @@ -87,34 +81,34 @@ def warp_triangle(img1, img2, t1, t2): r2 = cv2.boundingRect(np.float32([t2])) # Offset points by left top corner of the respective rectangles - t1Rect = [] - t2Rect = [] - t2RectInt = [] + t1_rect = [] + t2_rect = [] + t2_rect_int = [] for i in range(0, 3): - t1Rect.append(((t1[i][0] - r1[0]), (t1[i][1] - r1[1]))) - t2Rect.append(((t2[i][0] - r2[0]), (t2[i][1] - r2[1]))) - t2RectInt.append(((t2[i][0] - r2[0]), (t2[i][1] - r2[1]))) + t1_rect.append(((t1[i][0] - r1[0]), (t1[i][1] - r1[1]))) + t2_rect.append(((t2[i][0] - r2[0]), (t2[i][1] - r2[1]))) + t2_rect_int.append(((t2[i][0] - r2[0]), (t2[i][1] - r2[1]))) # Get mask by filling triangle mask = np.zeros((r2[3], r2[2], 3), dtype=np.float32) - cv2.fillConvexPoly(mask, np.int32(t2RectInt), (1.0, 1.0, 1.0), 16, 0); + cv2.fillConvexPoly(mask, np.int32(t2_rect_int), (1.0, 1.0, 1.0), 16, 0); # Apply warpImage to small rectangular patches - img1Rect = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]] - # img2Rect = np.zeros((r2[3], r2[2]), dtype = img1Rect.dtype) + img1_rect = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]] + # img2_rect = np.zeros((r2[3], r2[2]), dtype = img1_rect.dtype) size = (r2[2], r2[3]) - img2Rect = apply_affine_transform(img1Rect, t1Rect, t2Rect, size) + img2_rect = apply_affine_transform(img1_rect, t1_rect, t2_rect, size) - img2Rect = img2Rect * mask + img2_rect = img2_rect * mask # Copy triangular region of the rectangular patch to the output image img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] = img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] * ( (1.0, 1.0, 1.0) - mask) - img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] = img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] + img2Rect + img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] = img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]] + img2_rect def get_points(img): @@ -140,9 +134,9 @@ def get_points(img): def swap_faces(source1, source2): # Read images - img1 = cv2.imread(source1) - img2 = cv2.imread(source2) - img1Warped = np.copy(img2) + img1 = cv2.imread(os.path.join(UPLOAD_FOLDER, source1)) + img2 = cv2.imread(os.path.join(UPLOAD_FOLDER, source2)) + img1_warped = np.copy(img2) # Read array of corresponding points points1 = get_points(img1) @@ -152,15 +146,15 @@ def swap_faces(source1, source2): hull1 = [] hull2 = [] - hullIndex = cv2.convexHull(np.array(points2), returnPoints=False) + hull_index = cv2.convexHull(np.array(points2), returnPoints=False) - for i in range(0, len(hullIndex)): - hull1.append(points1[int(hullIndex[i])]) - hull2.append(points2[int(hullIndex[i])]) + for i in range(0, len(hull_index)): + hull1.append(points1[int(hull_index[i])]) + hull2.append(points2[int(hull_index[i])]) # Find delanauy traingulation for convex hull points - sizeImg2 = img2.shape - rect = (0, 0, sizeImg2[1], sizeImg2[0]) + size_img2 = img2.shape + rect = (0, 0, size_img2[1], size_img2[0]) dt = calculate_delaunay_triangles(rect, hull2) @@ -177,7 +171,7 @@ def swap_faces(source1, source2): t1.append(hull1[dt[i][j]]) t2.append(hull2[dt[i][j]]) - warp_triangle(img1, img1Warped, t1, t2) + warp_triangle(img1, img1_warped, t1, t2) # Calculate Mask hull8U = [] @@ -193,7 +187,7 @@ def swap_faces(source1, source2): center = (r[0] + int(r[2] / 2), r[1] + int(r[3] / 2)) # Clone seamlessly. - output = cv2.seamlessClone(np.uint8(img1Warped), img2, mask, center, cv2.NORMAL_CLONE) + output = cv2.seamlessClone(np.uint8(img1_warped), img2, mask, center, cv2.NORMAL_CLONE) cv2.imwrite(os.path.join(app.config['UPLOAD_FOLDER'], RESULT_IMG_NAME), output)