Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -9,141 +9,138 @@ import matplotlib.pyplot as plt
|
|
9 |
import numpy as np
|
10 |
import streamlit as st
|
11 |
|
12 |
-
def app():
|
13 |
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
|
19 |
-
|
20 |
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
st.markdown("")
|
30 |
|
31 |
st.markdown("")
|
32 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
33 |
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
)
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
fig, ax = plt.subplots()
|
119 |
-
ax.pie(x, colors=colors, radius=2, center=(4, 4),
|
120 |
-
wedgeprops={"linewidth": 1, "edgecolor": "white"}, frame=False,labels =list(x.index))
|
121 |
-
|
122 |
-
st.markdown("## π Anything related to SDGs?")
|
123 |
-
|
124 |
-
c4, c5, c6 = st.columns([5, 7, 1])
|
125 |
-
|
126 |
-
# Add styling
|
127 |
-
cmGreen = sns.light_palette("green", as_cmap=True)
|
128 |
-
cmRed = sns.light_palette("red", as_cmap=True)
|
129 |
-
df = df.style.background_gradient(
|
130 |
-
cmap=cmGreen,
|
131 |
-
subset=[
|
132 |
-
"Relevancy",
|
133 |
-
],
|
134 |
-
)
|
135 |
-
|
136 |
-
format_dictionary = {
|
137 |
-
"Relevancy": "{:.1%}",
|
138 |
-
}
|
139 |
-
|
140 |
-
df = df.format(format_dictionary)
|
141 |
-
|
142 |
-
with c4:
|
143 |
-
st.pyplot(fig)
|
144 |
-
with c5:
|
145 |
-
st.table(df)
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
app.run()
|
|
|
9 |
import numpy as np
|
10 |
import streamlit as st
|
11 |
|
|
|
12 |
|
13 |
+
with st.container():
|
14 |
+
st.markdown("<h1 style='text-align: center; color: black;'> Policy Action Tracking</h1>", unsafe_allow_html=True)
|
15 |
+
st.write(' ')
|
16 |
+
st.write(' ')
|
17 |
|
18 |
+
with st.expander("βΉοΈ - About this app", expanded=True):
|
19 |
|
20 |
+
st.write(
|
21 |
+
"""
|
22 |
+
The *Policy Action Tracker* app is an easy-to-use interface built in Streamlit for analyzing policy documents - developed by GIZ Data and the Sustainable Development Solution Network.
|
23 |
|
24 |
+
It uses a minimal keyword extraction technique that leverages multiple NLP embeddings and relies on [Transformers] (https://huggingface.co/transformers/) π€ to create keywords/keyphrases that are most similar to a document.
|
25 |
+
"""
|
26 |
+
)
|
|
|
|
|
27 |
|
28 |
st.markdown("")
|
29 |
+
|
30 |
+
st.markdown("")
|
31 |
+
st.markdown("## π Step One: Upload document ")
|
32 |
+
|
33 |
+
with st.container():
|
34 |
+
|
35 |
+
file = st.file_uploader('Upload PDF File', type=['pdf'])
|
36 |
+
|
37 |
+
if file is not None:
|
38 |
+
text = []
|
39 |
+
with pdfplumber.open(file) as pdf:
|
40 |
+
for page in pdf.pages:
|
41 |
+
text.append(page.extract_text())
|
42 |
+
text_str = ' '.join([page for page in text])
|
43 |
+
|
44 |
+
st.write('Number of pages:',len(pdf.pages))
|
45 |
+
|
46 |
+
@st.cache(allow_output_mutation=True)
|
47 |
+
def load_model():
|
48 |
+
return KeyBERT()
|
49 |
+
|
50 |
+
kw_model = load_model()
|
51 |
+
|
52 |
+
keywords = kw_model.extract_keywords(
|
53 |
+
text_str,
|
54 |
+
keyphrase_ngram_range=(1, 2),
|
55 |
+
use_mmr=True,
|
56 |
+
stop_words="english",
|
57 |
+
top_n=15,
|
58 |
+
diversity=0.7,
|
59 |
+
)
|
60 |
+
|
61 |
+
st.markdown("## π What is my document about?")
|
62 |
|
63 |
+
df = (
|
64 |
+
DataFrame(keywords, columns=["Keyword/Keyphrase", "Relevancy"])
|
65 |
+
.sort_values(by="Relevancy", ascending=False)
|
66 |
+
.reset_index(drop=True)
|
67 |
+
)
|
68 |
+
|
69 |
+
df.index += 1
|
70 |
+
|
71 |
+
# Add styling
|
72 |
+
cmGreen = sns.light_palette("green", as_cmap=True)
|
73 |
+
cmRed = sns.light_palette("red", as_cmap=True)
|
74 |
+
df = df.style.background_gradient(
|
75 |
+
cmap=cmGreen,
|
76 |
+
subset=[
|
77 |
+
"Relevancy",
|
78 |
+
],
|
79 |
+
)
|
80 |
+
c1, c2, c3 = st.columns([1, 3, 1])
|
81 |
+
|
82 |
+
format_dictionary = {
|
83 |
+
"Relevancy": "{:.1%}",
|
84 |
+
}
|
85 |
+
|
86 |
+
df = df.format(format_dictionary)
|
87 |
+
|
88 |
+
with c2:
|
89 |
+
st.table(df)
|
90 |
+
|
91 |
+
######## SDG!
|
92 |
+
from transformers import pipeline
|
93 |
+
|
94 |
+
finetuned_checkpoint = "peter2000/roberta-base-finetuned-osdg"
|
95 |
+
classifier = pipeline("text-classification", model=finetuned_checkpoint)
|
96 |
+
|
97 |
+
word_list = text_str.split()
|
98 |
+
len_word_list = len(word_list)
|
99 |
+
par_list = []
|
100 |
+
par_len = 130
|
101 |
+
for i in range(0,len_word_list // par_len):
|
102 |
+
string_part = ' '.join(word_list[i*par_len:(i+1)*par_len])
|
103 |
+
par_list.append(string_part)
|
104 |
+
|
105 |
+
labels = classifier(par_list)
|
106 |
+
labels_= [(l['label'],l['score']) for l in labels]
|
107 |
+
df = DataFrame(labels_, columns=["SDG", "Relevancy"])
|
108 |
+
df['text'] = par_list
|
109 |
+
df = df.sort_values(by="Relevancy", ascending=False).reset_index(drop=True)
|
110 |
+
df.index += 1
|
111 |
+
#df =df[df['Relevancy']>.95]
|
112 |
+
x = df['SDG'].value_counts()
|
113 |
+
|
114 |
+
plt.rcParams['font.size'] = 25
|
115 |
+
colors = plt.get_cmap('Blues')(np.linspace(0.2, 0.7, len(x)))
|
116 |
+
# plot
|
117 |
+
fig, ax = plt.subplots()
|
118 |
+
ax.pie(x, colors=colors, radius=2, center=(4, 4),
|
119 |
+
wedgeprops={"linewidth": 1, "edgecolor": "white"}, frame=False,labels =list(x.index))
|
120 |
+
|
121 |
+
st.markdown("## π Anything related to SDGs?")
|
122 |
+
|
123 |
+
c4, c5, c6 = st.columns([5, 7, 1])
|
124 |
+
|
125 |
+
# Add styling
|
126 |
+
cmGreen = sns.light_palette("green", as_cmap=True)
|
127 |
+
cmRed = sns.light_palette("red", as_cmap=True)
|
128 |
+
df = df.style.background_gradient(
|
129 |
+
cmap=cmGreen,
|
130 |
+
subset=[
|
131 |
+
"Relevancy",
|
132 |
+
],
|
133 |
+
)
|
134 |
+
|
135 |
+
format_dictionary = {
|
136 |
+
"Relevancy": "{:.1%}",
|
137 |
+
}
|
138 |
+
|
139 |
+
df = df.format(format_dictionary)
|
140 |
+
|
141 |
+
with c4:
|
142 |
+
st.pyplot(fig)
|
143 |
+
with c5:
|
144 |
+
st.table(df)
|
145 |
+
|
146 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|