Brasd99 commited on
Commit
bac067f
·
1 Parent(s): c9097a0

Code refactoring

Browse files
Files changed (1) hide show
  1. app.py +84 -85
app.py CHANGED
@@ -18,16 +18,16 @@ from reportlab.lib.styles import getSampleStyleSheet
18
  from reportlab.platypus import Image, SimpleDocTemplate, Table, TableStyle
19
  from ultralytics import YOLO
20
 
21
- with open('config.json', 'r') as f:
22
  config = json.load(f)
23
 
24
- FACE_DET_TRESH = config['FACE_DET_TRESH']
25
- FACE_DIST_TRESH = config['FACE_DIST_TRESH']
26
- YOLO_WEIGHTS_URL = config['YOLO_WEIGHTS_URL']
27
- AVATARS_URI = config['AVATARS_URI']
28
- APP_NAME = config['APP_NAME']
29
- APP_DESCRIPTION = config['APP_DESCRIPTION']
30
- APP_LOGO = config['APP_LOGO']
31
 
32
  def load_detector():
33
  yolo_weights_filename = os.path.basename(YOLO_WEIGHTS_URL)
@@ -43,18 +43,18 @@ model = load_detector()
43
 
44
  styles = getSampleStyleSheet()
45
  style_table = TableStyle([
46
- ('BACKGROUND', (0, 0), (-1, 0), colors.grey),
47
- ('TEXTCOLOR', (0, 0), (-1, 0), colors.whitesmoke),
48
- ('ALIGN', (0, 0), (-1, -1), 'CENTER'),
49
- ('FONTNAME', (0, 0), (-1, 0), 'Helvetica-Bold'),
50
- ('FONTSIZE', (0, 0), (-1, 0), 14),
51
- ('BOTTOMPADDING', (0, 0), (-1, 0), 12),
52
- ('BACKGROUND', (0, 1), (-1, -1), colors.beige),
53
- ('GRID', (0, 0), (-1, -1), 1, colors.black),
54
  ])
55
 
56
  def parse_album(data):
57
- album_info = data['NGRX_STATE']['game']['info']['data']['photoSetUrl']
58
  album_info = album_info.split("-")[-1].split("_")
59
  owner_id = - int(album_info[0])
60
  album_id = int(album_info[1])
@@ -62,27 +62,27 @@ def parse_album(data):
62
 
63
  def get_photos(owner_id, album_id, vk):
64
  offset = 0
65
- total_count = float('inf')
66
  count_per_request = 50
67
 
68
  output = []
69
 
70
  while offset < total_count:
71
  params = {
72
- 'owner_id': owner_id,
73
- 'album_id': album_id,
74
- 'count': count_per_request,
75
- 'offset': offset,
76
- 'extended': '1'
77
  }
78
 
79
  response = vk.photos.get(**params)
80
 
81
- for item in response['items']:
82
- max_item = max(item['sizes'], key=lambda item: item['height'])
83
- output.append(max_item['url'])
84
 
85
- total_count = response['count']
86
  offset += count_per_request
87
 
88
  return output
@@ -92,41 +92,41 @@ def download_images(photos, players):
92
  folder_name = current_datetime.strftime("%Y-%m-%d_%H-%M-%S")
93
  os.mkdir(folder_name)
94
 
95
- players_path = os.path.join(folder_name, 'players')
96
- photos_path = os.path.join(folder_name, 'photos')
97
- temp_path = os.path.join(folder_name, 'temp')
98
 
99
  os.mkdir(players_path)
100
  os.mkdir(photos_path)
101
  os.mkdir(temp_path)
102
 
103
- update_progress(0, 'Downloading photos...')
104
  for i, photo_url in enumerate(photos):
105
- filename = f'{i}.jpg'
106
  response = requests.get(photo_url)
107
  with open(os.path.join(photos_path, filename), "wb") as file:
108
  file.write(response.content)
109
- update_progress((i+1)/len(photos), 'Downloading photos...')
110
 
111
  for team_state in players.keys():
112
  update_progress(0, f"Downloading {team_state} players' avatars...")
113
  for i, player in enumerate(players[team_state]):
114
  filename = f"{player['id']}.jpg"
115
- response = requests.get(player['avatar_url'])
116
  with open(os.path.join(players_path, filename), "wb") as file:
117
  file.write(response.content)
118
  update_progress((i+1)/len(players[team_state]), f"Downloading {team_state} players' avatars...")
119
 
120
  return {
121
- 'photos_path': photos_path,
122
- 'players_path': players_path,
123
- 'temp_path': temp_path,
124
- 'folder_name': folder_name
125
  }
126
 
127
  def find_photos(data, vk):
128
- pattern = re.compile('<script id="axl-desktop-state" type="application/json">(.+?)</script>')
129
- script_content = pattern.search(data).group(1).replace('&q;', '"')
130
 
131
  data = json.loads(script_content)
132
 
@@ -136,52 +136,52 @@ def find_photos(data, vk):
136
 
137
  def translate(text):
138
  translator = Translator()
139
- output = translator.translate(text, src='ru', dest='en')
140
  return output.text
141
 
142
  def get_players(data):
143
  output = {}
144
- team_states = ['home', 'away']
145
- soup = BeautifulSoup(data, 'lxml')
146
 
147
  for team_state in team_states:
148
- update_progress(0, f'Getting information about {team_state} players...')
149
  output[team_state] = []
150
  player_roots = soup.find_all("div", {"class": f"{team_state} ng-star-inserted"})
151
  for i, player_root in enumerate(player_roots):
152
  player_info = player_root.find("a", {"class": "wrapper ng-star-inserted"})
153
- id = re.findall(r'\d+', player_info['href'])[-1]
154
  avatar_url = AVATARS_URI.replace("PLAYER_ID", id)
155
  name = player_info.find("span", {"class": "name"}).get_text()
156
  name = translate(name)
157
  position = player_info.find("span", {"class": "position"}).get_text()
158
  output[team_state].append({
159
- 'id': id,
160
- 'name': name,
161
- 'position': position,
162
- 'avatar_url': avatar_url
163
  })
164
- update_progress((i+1)/len(player_roots), f'Getting information about {team_state} players...')
165
 
166
  return output
167
 
168
  def load_players_avatars(players, images_path, face_det_tresh):
169
  for team_state in players.keys():
170
- update_progress(0, f'Reading avatars of {team_state} team...')
171
  for i, player in enumerate(players[team_state]):
172
  image_name = f"{player['id']}.jpg"
173
- player['image'] = read_image_from_path(os.path.join(images_path, image_name))
174
- faces = find_faces(player['image'], face_det_tresh)
175
  if faces:
176
- player['face'] = faces[0]
177
 
178
- update_progress((i+1)/len(players[team_state]), f'Reading avatars of {team_state} team...')
179
 
180
  return players
181
 
182
  def find_distance(base_face, check_face):
183
  result = DeepFace.verify(base_face, check_face, enforce_detection=False)
184
- return result['distance']
185
 
186
  def read_image_from_path(path):
187
  return cv2.imread(path)
@@ -190,7 +190,7 @@ def read_images_from_path(path):
190
  images = []
191
 
192
  files = os.listdir(path)
193
- update_progress(0, 'Reading photos...')
194
 
195
  for i, filename in enumerate(files):
196
  if filename.endswith(".jpg"):
@@ -199,7 +199,7 @@ def read_images_from_path(path):
199
  if image is not None:
200
  images.append(image)
201
 
202
- update_progress((i+1)/len(files), 'Reading photos...')
203
 
204
  return images
205
 
@@ -225,10 +225,10 @@ def find_faces(image, face_det_tresh):
225
  def is_face_exists(players, face, face_dist_tresh):
226
  for team_state in players.keys():
227
  for player in players[team_state]:
228
- if 'face' in player:
229
- distance = find_distance(player['face'], face)
230
  if distance <= face_dist_tresh:
231
- return player['id'], player['face']
232
  return None, None
233
 
234
  def add_players_table(elements, players):
@@ -238,12 +238,12 @@ def add_players_table(elements, players):
238
  for team_state in players.keys():
239
  update_progress(0, f"Creating dump of {team_state}'s squad...")
240
  for i, player in enumerate(players[team_state]):
241
- face = cv2_to_reportlab(player['face']) if 'face' in player else None
242
- avatar = cv2_to_reportlab(player['image'])
243
  line = [
244
- player['id'],
245
- player['name'],
246
- player['position'],
247
  avatar,
248
  face
249
  ]
@@ -261,7 +261,7 @@ def check_faces(elements, photos, players, face_det_tresh, face_dist_tresh):
261
  ["Face", "Player ID", "Player Face"]
262
  ]
263
 
264
- update_progress(0, 'Comparing faces...')
265
  for i, photo in enumerate(photos):
266
  faces = find_faces(photo, face_det_tresh)
267
  for j, face in enumerate(faces):
@@ -272,7 +272,7 @@ def check_faces(elements, photos, players, face_det_tresh, face_dist_tresh):
272
  player_face = cv2_to_reportlab(player_face)
273
  tmp_arr.append(player_face)
274
  data.append(tmp_arr)
275
- update_progress((j+1)/len(faces), f'[{i + 1}/{len(photos)}] Comparing faces...')
276
 
277
  table = Table(data)
278
  table.setStyle(style_table)
@@ -285,27 +285,26 @@ def update_progress(percent, description):
285
  progress_bar.progress(percent)
286
  progress_status_text.text(description)
287
 
288
-
289
  def process(token, afl_link, face_dist_tresh, face_det_tresh):
290
- update_progress(0, 'Connecting to vk...')
291
  vk_session = vk_api.VkApi(token=token)
292
  vk = vk_session.get_api()
293
- update_progress(100, 'Connected to vk')
294
 
295
- update_progress(0, 'Getting information from afl...')
296
  response = requests.get(afl_link)
297
- update_progress(100, 'Got information from afl')
298
 
299
- update_progress(0, 'Getting information about photos...')
300
  photos = find_photos(response.text, vk)
301
- update_progress(100, 'Got information about photos')
302
 
303
  players = get_players(response.text)
304
  result = download_images(photos, players)
305
- photos = read_images_from_path(result['photos_path'])
306
- players = load_players_avatars(players, result['players_path'], face_det_tresh)
307
 
308
- table_file = os.path.join(result['temp_path'], 'table.pdf')
309
 
310
  doc = SimpleDocTemplate(table_file, pagesize=letter)
311
 
@@ -315,10 +314,10 @@ def process(token, afl_link, face_dist_tresh, face_det_tresh):
315
 
316
  doc.build(elements)
317
 
318
- with open(table_file, 'rb') as file:
319
  pdf_bytes = file.read()
320
 
321
- shutil.rmtree(result['folder_name'])
322
 
323
  return pdf_bytes
324
 
@@ -329,11 +328,11 @@ st.image(APP_LOGO, use_column_width=True)
329
 
330
  st.write(APP_DESCRIPTION)
331
 
332
- access_token = st.text_input("Your VK API access token", help='You can obtain your token from https://vkhost.github.io/')
333
- afl_url = st.text_input("AFL url", help='Example: https://afl.ru/football/afl-moscow-8x8/afl-cup-krasnaya-presnya-3097/matches/463676')
334
 
335
- face_det_tresh = st.slider('face_det_tresh:', 0.0, 1.0, FACE_DET_TRESH, 0.01, help='Adjust the threshold value for face detection.')
336
- face_dist_tresh = st.slider('face_dist_tresh:', 0.0, 1.0, FACE_DIST_TRESH, 0.01, help='Adjust the threshold to determine the minimum acceptable distance between faces.')
337
 
338
  button_clicked = st.button("Process")
339
 
@@ -341,4 +340,4 @@ if button_clicked:
341
  progress_bar = st.progress(0)
342
  progress_status_text = st.empty()
343
  pdf_bytes = process(access_token, afl_url, face_dist_tresh, face_det_tresh)
344
- st.download_button(label='Download PDF', data=pdf_bytes, file_name='output.pdf')
 
18
  from reportlab.platypus import Image, SimpleDocTemplate, Table, TableStyle
19
  from ultralytics import YOLO
20
 
21
+ with open("config.json", "r") as f:
22
  config = json.load(f)
23
 
24
+ FACE_DET_TRESH = config["FACE_DET_TRESH"]
25
+ FACE_DIST_TRESH = config["FACE_DIST_TRESH"]
26
+ YOLO_WEIGHTS_URL = config["YOLO_WEIGHTS_URL"]
27
+ AVATARS_URI = config["AVATARS_URI"]
28
+ APP_NAME = config["APP_NAME"]
29
+ APP_DESCRIPTION = config["APP_DESCRIPTION"]
30
+ APP_LOGO = config["APP_LOGO"]
31
 
32
  def load_detector():
33
  yolo_weights_filename = os.path.basename(YOLO_WEIGHTS_URL)
 
43
 
44
  styles = getSampleStyleSheet()
45
  style_table = TableStyle([
46
+ ("BACKGROUND", (0, 0), (-1, 0), colors.grey),
47
+ ("TEXTCOLOR", (0, 0), (-1, 0), colors.whitesmoke),
48
+ ("ALIGN", (0, 0), (-1, -1), "CENTER"),
49
+ ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"),
50
+ ("FONTSIZE", (0, 0), (-1, 0), 14),
51
+ ("BOTTOMPADDING", (0, 0), (-1, 0), 12),
52
+ ("BACKGROUND", (0, 1), (-1, -1), colors.beige),
53
+ ("GRID", (0, 0), (-1, -1), 1, colors.black),
54
  ])
55
 
56
  def parse_album(data):
57
+ album_info = data["NGRX_STATE"]["game"]["info"]["data"]["photoSetUrl"]
58
  album_info = album_info.split("-")[-1].split("_")
59
  owner_id = - int(album_info[0])
60
  album_id = int(album_info[1])
 
62
 
63
  def get_photos(owner_id, album_id, vk):
64
  offset = 0
65
+ total_count = float("inf")
66
  count_per_request = 50
67
 
68
  output = []
69
 
70
  while offset < total_count:
71
  params = {
72
+ "owner_id": owner_id,
73
+ "album_id": album_id,
74
+ "count": count_per_request,
75
+ "offset": offset,
76
+ "extended": "1"
77
  }
78
 
79
  response = vk.photos.get(**params)
80
 
81
+ for item in response["items"]:
82
+ max_item = max(item["sizes"], key=lambda item: item["height"])
83
+ output.append(max_item["url"])
84
 
85
+ total_count = response["count"]
86
  offset += count_per_request
87
 
88
  return output
 
92
  folder_name = current_datetime.strftime("%Y-%m-%d_%H-%M-%S")
93
  os.mkdir(folder_name)
94
 
95
+ players_path = os.path.join(folder_name, "players")
96
+ photos_path = os.path.join(folder_name, "photos")
97
+ temp_path = os.path.join(folder_name, "temp")
98
 
99
  os.mkdir(players_path)
100
  os.mkdir(photos_path)
101
  os.mkdir(temp_path)
102
 
103
+ update_progress(0, "Downloading photos...")
104
  for i, photo_url in enumerate(photos):
105
+ filename = f"{i}.jpg"
106
  response = requests.get(photo_url)
107
  with open(os.path.join(photos_path, filename), "wb") as file:
108
  file.write(response.content)
109
+ update_progress((i+1)/len(photos), "Downloading photos...")
110
 
111
  for team_state in players.keys():
112
  update_progress(0, f"Downloading {team_state} players' avatars...")
113
  for i, player in enumerate(players[team_state]):
114
  filename = f"{player['id']}.jpg"
115
+ response = requests.get(player["avatar_url"])
116
  with open(os.path.join(players_path, filename), "wb") as file:
117
  file.write(response.content)
118
  update_progress((i+1)/len(players[team_state]), f"Downloading {team_state} players' avatars...")
119
 
120
  return {
121
+ "photos_path": photos_path,
122
+ "players_path": players_path,
123
+ "temp_path": temp_path,
124
+ "folder_name": folder_name
125
  }
126
 
127
  def find_photos(data, vk):
128
+ pattern = re.compile("<script id='axl-desktop-state' type='application/json'>(.+?)</script>")
129
+ script_content = pattern.search(data).group(1).replace("&q;", "")
130
 
131
  data = json.loads(script_content)
132
 
 
136
 
137
  def translate(text):
138
  translator = Translator()
139
+ output = translator.translate(text, src="ru", dest="en")
140
  return output.text
141
 
142
  def get_players(data):
143
  output = {}
144
+ team_states = ["home", "away"]
145
+ soup = BeautifulSoup(data, "lxml")
146
 
147
  for team_state in team_states:
148
+ update_progress(0, f"Getting information about {team_state} players...")
149
  output[team_state] = []
150
  player_roots = soup.find_all("div", {"class": f"{team_state} ng-star-inserted"})
151
  for i, player_root in enumerate(player_roots):
152
  player_info = player_root.find("a", {"class": "wrapper ng-star-inserted"})
153
+ id = re.findall(r"\d+", player_info["href"])[-1]
154
  avatar_url = AVATARS_URI.replace("PLAYER_ID", id)
155
  name = player_info.find("span", {"class": "name"}).get_text()
156
  name = translate(name)
157
  position = player_info.find("span", {"class": "position"}).get_text()
158
  output[team_state].append({
159
+ "id": id,
160
+ "name": name,
161
+ "position": position,
162
+ "avatar_url": avatar_url
163
  })
164
+ update_progress((i+1)/len(player_roots), f"Getting information about {team_state} players...")
165
 
166
  return output
167
 
168
  def load_players_avatars(players, images_path, face_det_tresh):
169
  for team_state in players.keys():
170
+ update_progress(0, f"Reading avatars of {team_state} team...")
171
  for i, player in enumerate(players[team_state]):
172
  image_name = f"{player['id']}.jpg"
173
+ player["image"] = read_image_from_path(os.path.join(images_path, image_name))
174
+ faces = find_faces(player["image"], face_det_tresh)
175
  if faces:
176
+ player["face"] = faces[0]
177
 
178
+ update_progress((i+1)/len(players[team_state]), f"Reading avatars of {team_state} team...")
179
 
180
  return players
181
 
182
  def find_distance(base_face, check_face):
183
  result = DeepFace.verify(base_face, check_face, enforce_detection=False)
184
+ return result["distance"]
185
 
186
  def read_image_from_path(path):
187
  return cv2.imread(path)
 
190
  images = []
191
 
192
  files = os.listdir(path)
193
+ update_progress(0, "Reading photos...")
194
 
195
  for i, filename in enumerate(files):
196
  if filename.endswith(".jpg"):
 
199
  if image is not None:
200
  images.append(image)
201
 
202
+ update_progress((i+1)/len(files), "Reading photos...")
203
 
204
  return images
205
 
 
225
  def is_face_exists(players, face, face_dist_tresh):
226
  for team_state in players.keys():
227
  for player in players[team_state]:
228
+ if "face" in player:
229
+ distance = find_distance(player["face"], face)
230
  if distance <= face_dist_tresh:
231
+ return player["id"], player["face"]
232
  return None, None
233
 
234
  def add_players_table(elements, players):
 
238
  for team_state in players.keys():
239
  update_progress(0, f"Creating dump of {team_state}'s squad...")
240
  for i, player in enumerate(players[team_state]):
241
+ face = cv2_to_reportlab(player["face"]) if "face" in player else None
242
+ avatar = cv2_to_reportlab(player["image"])
243
  line = [
244
+ player["id"],
245
+ player["name"],
246
+ player["position"],
247
  avatar,
248
  face
249
  ]
 
261
  ["Face", "Player ID", "Player Face"]
262
  ]
263
 
264
+ update_progress(0, "Comparing faces...")
265
  for i, photo in enumerate(photos):
266
  faces = find_faces(photo, face_det_tresh)
267
  for j, face in enumerate(faces):
 
272
  player_face = cv2_to_reportlab(player_face)
273
  tmp_arr.append(player_face)
274
  data.append(tmp_arr)
275
+ update_progress((j+1)/len(faces), f"[{i + 1}/{len(photos)}] Comparing faces...")
276
 
277
  table = Table(data)
278
  table.setStyle(style_table)
 
285
  progress_bar.progress(percent)
286
  progress_status_text.text(description)
287
 
 
288
  def process(token, afl_link, face_dist_tresh, face_det_tresh):
289
+ update_progress(0, "Connecting to vk...")
290
  vk_session = vk_api.VkApi(token=token)
291
  vk = vk_session.get_api()
292
+ update_progress(100, "Connected to vk")
293
 
294
+ update_progress(0, "Getting information from afl...")
295
  response = requests.get(afl_link)
296
+ update_progress(100, "Got information from afl")
297
 
298
+ update_progress(0, "Getting information about photos...")
299
  photos = find_photos(response.text, vk)
300
+ update_progress(100, "Got information about photos")
301
 
302
  players = get_players(response.text)
303
  result = download_images(photos, players)
304
+ photos = read_images_from_path(result["photos_path"])
305
+ players = load_players_avatars(players, result["players_path"], face_det_tresh)
306
 
307
+ table_file = os.path.join(result["temp_path"], "table.pdf")
308
 
309
  doc = SimpleDocTemplate(table_file, pagesize=letter)
310
 
 
314
 
315
  doc.build(elements)
316
 
317
+ with open(table_file, "rb") as file:
318
  pdf_bytes = file.read()
319
 
320
+ shutil.rmtree(result["folder_name"])
321
 
322
  return pdf_bytes
323
 
 
328
 
329
  st.write(APP_DESCRIPTION)
330
 
331
+ access_token = st.text_input("Your VK API access token", help="You can obtain your token from https://vkhost.github.io/")
332
+ afl_url = st.text_input("AFL url", help="Example: https://afl.ru/football/afl-moscow-8x8/afl-cup-krasnaya-presnya-3097/matches/463676")
333
 
334
+ face_det_tresh = st.slider("face_det_tresh:", 0.0, 1.0, FACE_DET_TRESH, 0.01, help="Adjust the threshold value for face detection.")
335
+ face_dist_tresh = st.slider("face_dist_tresh:", 0.0, 1.0, FACE_DIST_TRESH, 0.01, help="Adjust the threshold to determine the minimum acceptable distance between faces.")
336
 
337
  button_clicked = st.button("Process")
338
 
 
340
  progress_bar = st.progress(0)
341
  progress_status_text = st.empty()
342
  pdf_bytes = process(access_token, afl_url, face_dist_tresh, face_det_tresh)
343
+ st.download_button(label="Download PDF", data=pdf_bytes, file_name="output.pdf")