kambris commited on
Commit
81e4873
ยท
verified ยท
1 Parent(s): a165f87

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +17 -1
app.py CHANGED
@@ -13,7 +13,12 @@ import pkg_resources
13
  import folium
14
  import country_converter as coco
15
  import time
 
16
 
 
 
 
 
17
 
18
  current_dir = os.path.dirname(os.path.abspath(__file__))
19
  font_path = os.path.join(current_dir, "ArabicR2013-J25x.ttf")
@@ -33,7 +38,7 @@ ARABIC_STOP_WORDS = {
33
  'ู…ู†ุฐ', 'ู†ูุณ', 'ุญูŠุซ', 'ู‡ู†ุงูƒ', 'ุฌุฏุง', 'ุฐุงุช', 'ุถู…ู†', 'ุงู†ู‡', 'ู„ุฏู‰',
34
  'ุนู„ูŠู‡', 'ู…ุซู„', 'ุฃู…ุง', 'ู„ุฏูŠ', 'ููŠู‡', 'ูƒู„ู…', 'ู„ูƒู†', 'ุงูŠุถุง', 'ู„ุงุฒู…',
35
  'ูŠุฌุจ', 'ุตุงุฑ', 'ุตุงุฑุช', 'ุถุฏ', 'ูŠุง', 'ู„ุง', 'ุงู…ุง',
36
- 'ุจู‡ุง', 'ุงู†', 'ุจู‡', 'ุงู„ูŠ', 'ู„ู…ุง', 'ุงู†ุง', 'ุงู„ูŠูƒ', 'ู„ูŠ', 'ู„ูƒ', 'ู‚ู„ุช',
37
 
38
  # Middle group prefixed with "ูˆ"
39
  'ูˆุซู…', 'ูˆุฃูˆ', 'ูˆู„', 'ูˆุจ', 'ูˆูƒ', 'ูˆู„ู„', 'ูˆุงู„',
@@ -402,6 +407,10 @@ def process_and_summarize(df, bert_tokenizer, bert_model, emotion_classifier, to
402
  all_emotions = []
403
 
404
  embeddings = []
 
 
 
 
405
  for i, text in enumerate(texts):
406
  try:
407
  embedding = get_embedding_for_text(text, bert_tokenizer, bert_model)
@@ -413,6 +422,9 @@ def process_and_summarize(df, bert_tokenizer, bert_model, emotion_classifier, to
413
  except Exception as e:
414
  st.warning(f"Error generating embedding for text {i+1} in {country}: {str(e)}")
415
  continue
 
 
 
416
  progress = (i + 1) / len(texts) * 0.4
417
  progress_bar.progress(progress, text=f"Generated embeddings for {i+1}/{len(texts)} poems...")
418
 
@@ -420,9 +432,13 @@ def process_and_summarize(df, bert_tokenizer, bert_model, emotion_classifier, to
420
  texts = texts[:len(embeddings)]
421
  embeddings = np.array(embeddings)
422
 
 
 
423
  for i, text in enumerate(texts):
424
  emotion = classify_emotion(text, emotion_classifier)
425
  all_emotions.append(emotion)
 
 
426
  progress = 0.4 + ((i + 1) / len(texts) * 0.3)
427
  progress_bar.progress(progress, text=f"Classified emotions for {i+1}/{len(texts)} poems...")
428
 
 
13
  import folium
14
  import country_converter as coco
15
  import time
16
+ import gc
17
 
18
+ def clear_memory():
19
+ if torch.cuda.is_available():
20
+ torch.cuda.empty_cache()
21
+ gc.collect()
22
 
23
  current_dir = os.path.dirname(os.path.abspath(__file__))
24
  font_path = os.path.join(current_dir, "ArabicR2013-J25x.ttf")
 
38
  'ู…ู†ุฐ', 'ู†ูุณ', 'ุญูŠุซ', 'ู‡ู†ุงูƒ', 'ุฌุฏุง', 'ุฐุงุช', 'ุถู…ู†', 'ุงู†ู‡', 'ู„ุฏู‰',
39
  'ุนู„ูŠู‡', 'ู…ุซู„', 'ุฃู…ุง', 'ู„ุฏูŠ', 'ููŠู‡', 'ูƒู„ู…', 'ู„ูƒู†', 'ุงูŠุถุง', 'ู„ุงุฒู…',
40
  'ูŠุฌุจ', 'ุตุงุฑ', 'ุตุงุฑุช', 'ุถุฏ', 'ูŠุง', 'ู„ุง', 'ุงู…ุง',
41
+ 'ุจู‡ุง', 'ุงู†', 'ุจู‡', 'ุงู„ูŠ', 'ู„ู…ุง', 'ุงู†ุง', 'ุงู„ูŠูƒ', 'ู„ูŠ', 'ู„ูƒ','ุงุฐุง','ุจู„ุง','ุงูˆ','ู„ุฏูŠูƒ','ู„ุฏูŠู‡', 'ู‚ู„ุช',
42
 
43
  # Middle group prefixed with "ูˆ"
44
  'ูˆุซู…', 'ูˆุฃูˆ', 'ูˆู„', 'ูˆุจ', 'ูˆูƒ', 'ูˆู„ู„', 'ูˆุงู„',
 
407
  all_emotions = []
408
 
409
  embeddings = []
410
+
411
+ clear_memory()
412
+
413
+
414
  for i, text in enumerate(texts):
415
  try:
416
  embedding = get_embedding_for_text(text, bert_tokenizer, bert_model)
 
422
  except Exception as e:
423
  st.warning(f"Error generating embedding for text {i+1} in {country}: {str(e)}")
424
  continue
425
+ if i % 10 == 0:
426
+ clear_memory()
427
+
428
  progress = (i + 1) / len(texts) * 0.4
429
  progress_bar.progress(progress, text=f"Generated embeddings for {i+1}/{len(texts)} poems...")
430
 
 
432
  texts = texts[:len(embeddings)]
433
  embeddings = np.array(embeddings)
434
 
435
+ clear_memory()
436
+
437
  for i, text in enumerate(texts):
438
  emotion = classify_emotion(text, emotion_classifier)
439
  all_emotions.append(emotion)
440
+ if i % 10 == 0:
441
+ clear_memory()
442
  progress = 0.4 + ((i + 1) / len(texts) * 0.3)
443
  progress_bar.progress(progress, text=f"Classified emotions for {i+1}/{len(texts)} poems...")
444