Update app.py
Browse files
app.py
CHANGED
@@ -8,15 +8,10 @@ from io import StringIO
|
|
8 |
import csv
|
9 |
|
10 |
def get_backlinks(api_login, api_key, target_url, filters):
|
11 |
-
# Encoding credentials
|
12 |
encoded_credentials = base64.b64encode(f"{api_login}:{api_key}".encode()).decode()
|
13 |
-
|
14 |
-
# Setting headers with Basic Authentication
|
15 |
headers = {
|
16 |
'Authorization': f'Basic {encoded_credentials}'
|
17 |
}
|
18 |
-
|
19 |
-
# Prepare post data
|
20 |
post_data = {
|
21 |
0: {
|
22 |
"target": target_url,
|
@@ -25,24 +20,13 @@ def get_backlinks(api_login, api_key, target_url, filters):
|
|
25 |
"filters": filters
|
26 |
}
|
27 |
}
|
28 |
-
|
29 |
-
# Making the API request
|
30 |
response = requests.post("https://api.dataforseo.com/v3/backlinks/backlinks/live", json=post_data, headers=headers)
|
31 |
-
|
32 |
-
response_json = response.json()
|
33 |
-
|
34 |
-
# Check if the response contains 'results' key and handle the JSON structure appropriately
|
35 |
if response.status_code == 200:
|
36 |
response_data = response.json()
|
37 |
-
|
38 |
if 'tasks' in response_data:
|
39 |
-
# Assuming there is only one task and one result within each task
|
40 |
task_result = response_data['tasks'][0]['result']
|
41 |
if task_result and 'items' in task_result[0]:
|
42 |
-
# The actual backlink items are nested within 'items'
|
43 |
items = task_result[0]['items']
|
44 |
-
|
45 |
-
# Convert to DataFrame
|
46 |
df = pd.json_normalize(items)
|
47 |
return df
|
48 |
else:
|
@@ -57,10 +41,8 @@ def get_backlinks(api_login, api_key, target_url, filters):
|
|
57 |
return None
|
58 |
|
59 |
def convert_df_to_csv(df):
|
60 |
-
# Convert DataFrame to CSV
|
61 |
return df.to_csv(index=False).encode('utf-8')
|
62 |
-
|
63 |
-
# Worker thread to handle concurrent API calls
|
64 |
class BacklinkWorker(threading.Thread):
|
65 |
def __init__(self, jobs, results, api_login, api_key):
|
66 |
super().__init__()
|
@@ -78,18 +60,15 @@ class BacklinkWorker(threading.Thread):
|
|
78 |
df = get_backlinks(self.api_login, self.api_key, target_url, filters)
|
79 |
self.results.put((url_id, df))
|
80 |
|
81 |
-
# Function to process pasted data
|
82 |
def process_pasted_data(data):
|
83 |
data_io = StringIO(data.strip())
|
84 |
reader = csv.reader(data_io, delimiter='\n', quotechar='"')
|
85 |
return [row[0] for row in reader]
|
86 |
|
87 |
-
# Streamlit layout
|
88 |
st.sidebar.title("DataForSEO API Parameters")
|
89 |
api_login = st.sidebar.text_input("API Login", value="[email protected]")
|
90 |
api_key = st.sidebar.text_input("API Key", type="password")
|
91 |
|
92 |
-
# Filters input
|
93 |
st.sidebar.title("Filters")
|
94 |
st.sidebar.info ("A maximum of 8 filters are allowed through the API. That includes one for every comma-separated value below, one for every value not equal to 0 in the boxes, one for language, and one for each button ticked.")
|
95 |
url_from_not_contain = st.sidebar.text_input("URL from does not contain (comma-separated)", value="keyword,blogspot,/search,/tag")
|
@@ -140,14 +119,12 @@ filters.append("and")
|
|
140 |
if filters and filters[-1] == "and":
|
141 |
filters.pop()
|
142 |
|
143 |
-
# Sidebar options for concurrency and pasting data
|
144 |
num_concurrent_calls = st.sidebar.number_input("Concurrent Calls:", min_value=1, max_value=10, value=5, step=1)
|
145 |
data_section = st.sidebar.text_area("Paste Data:")
|
146 |
paste_data = st.sidebar.button("Paste Data")
|
147 |
add_row = st.sidebar.button("Add Row")
|
148 |
reset = st.sidebar.button("Reset")
|
149 |
|
150 |
-
# Managing rows and pasted data
|
151 |
if paste_data:
|
152 |
pasted_urls = process_pasted_data(data_section)
|
153 |
for i, url in enumerate(pasted_urls):
|
@@ -161,74 +138,42 @@ if add_row:
|
|
161 |
if reset:
|
162 |
st.session_state.clear()
|
163 |
|
164 |
-
# Generate and reset button logic
|
165 |
-
generate_pressed = st.sidebar.button("Generate All")
|
166 |
-
if generate_pressed:
|
167 |
-
st.session_state['generate_pressed'] = True
|
168 |
-
|
169 |
-
# Main app layout
|
170 |
row_count = st.session_state.get("row_count", 1)
|
171 |
-
|
172 |
-
|
173 |
-
download_placeholders = [st.empty() for _ in range(row_count)]
|
174 |
-
|
175 |
-
for i, (col1_placeholder, col2_placeholder) in enumerate(placeholders):
|
176 |
target_url_key = f"target_url_{i}"
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
182 |
jobs = Queue()
|
183 |
results = Queue()
|
184 |
workers = [BacklinkWorker(jobs, results, api_login, api_key) for _ in range(num_concurrent_calls)]
|
185 |
-
|
186 |
for worker in workers:
|
187 |
worker.start()
|
188 |
-
|
189 |
for i in range(row_count):
|
190 |
target_url = st.session_state.get(f"target_url_{i}", "")
|
191 |
if target_url:
|
192 |
jobs.put((i, target_url, filters))
|
193 |
-
|
194 |
for _ in workers:
|
195 |
jobs.put(None)
|
196 |
-
|
197 |
for worker in workers:
|
198 |
worker.join()
|
199 |
-
|
200 |
while not results.empty():
|
201 |
url_id, df = results.get()
|
202 |
st.session_state[f"df_{url_id}"] = df
|
203 |
-
|
204 |
-
# Update placeholders in the second column
|
205 |
-
for i, (_, col2_placeholder) in enumerate(placeholders):
|
206 |
-
df_key = f"df_{i}"
|
207 |
-
df = st.session_state.get(df_key)
|
208 |
-
|
209 |
-
with col2_placeholder.container():
|
210 |
-
if df is not None:
|
211 |
-
csv = convert_df_to_csv(df)
|
212 |
-
st.download_button(
|
213 |
-
label=f"Download data as CSV for URL {i + 1}",
|
214 |
-
data=csv,
|
215 |
-
file_name=f'backlinks_{i + 1}.csv',
|
216 |
-
mime='text/csv',
|
217 |
-
)
|
218 |
-
else:
|
219 |
-
st.error(f"Failed to generate CSV for URL {i + 1}: No data returned from the API or data processing error.")
|
220 |
-
|
221 |
-
# Display and download logic for each row
|
222 |
-
for i in range(row_count):
|
223 |
-
df_key = f"df_{i}"
|
224 |
-
df = st.session_state.get(df_key)
|
225 |
-
if df is not None:
|
226 |
-
csv = convert_df_to_csv(df)
|
227 |
-
st.download_button(
|
228 |
-
label=f"Download data as CSV for URL {i + 1}",
|
229 |
-
data=csv,
|
230 |
-
file_name=f'backlinks_{i + 1}.csv',
|
231 |
-
mime='text/csv',
|
232 |
-
)
|
233 |
-
elif df is None and generate_button:
|
234 |
-
st.error(f"Failed to generate CSV for URL {i + 1}: No data returned from the API or data processing error.")
|
|
|
8 |
import csv
|
9 |
|
10 |
def get_backlinks(api_login, api_key, target_url, filters):
|
|
|
11 |
encoded_credentials = base64.b64encode(f"{api_login}:{api_key}".encode()).decode()
|
|
|
|
|
12 |
headers = {
|
13 |
'Authorization': f'Basic {encoded_credentials}'
|
14 |
}
|
|
|
|
|
15 |
post_data = {
|
16 |
0: {
|
17 |
"target": target_url,
|
|
|
20 |
"filters": filters
|
21 |
}
|
22 |
}
|
|
|
|
|
23 |
response = requests.post("https://api.dataforseo.com/v3/backlinks/backlinks/live", json=post_data, headers=headers)
|
|
|
|
|
|
|
|
|
24 |
if response.status_code == 200:
|
25 |
response_data = response.json()
|
|
|
26 |
if 'tasks' in response_data:
|
|
|
27 |
task_result = response_data['tasks'][0]['result']
|
28 |
if task_result and 'items' in task_result[0]:
|
|
|
29 |
items = task_result[0]['items']
|
|
|
|
|
30 |
df = pd.json_normalize(items)
|
31 |
return df
|
32 |
else:
|
|
|
41 |
return None
|
42 |
|
43 |
def convert_df_to_csv(df):
|
|
|
44 |
return df.to_csv(index=False).encode('utf-8')
|
45 |
+
|
|
|
46 |
class BacklinkWorker(threading.Thread):
|
47 |
def __init__(self, jobs, results, api_login, api_key):
|
48 |
super().__init__()
|
|
|
60 |
df = get_backlinks(self.api_login, self.api_key, target_url, filters)
|
61 |
self.results.put((url_id, df))
|
62 |
|
|
|
63 |
def process_pasted_data(data):
|
64 |
data_io = StringIO(data.strip())
|
65 |
reader = csv.reader(data_io, delimiter='\n', quotechar='"')
|
66 |
return [row[0] for row in reader]
|
67 |
|
|
|
68 |
st.sidebar.title("DataForSEO API Parameters")
|
69 |
api_login = st.sidebar.text_input("API Login", value="[email protected]")
|
70 |
api_key = st.sidebar.text_input("API Key", type="password")
|
71 |
|
|
|
72 |
st.sidebar.title("Filters")
|
73 |
st.sidebar.info ("A maximum of 8 filters are allowed through the API. That includes one for every comma-separated value below, one for every value not equal to 0 in the boxes, one for language, and one for each button ticked.")
|
74 |
url_from_not_contain = st.sidebar.text_input("URL from does not contain (comma-separated)", value="keyword,blogspot,/search,/tag")
|
|
|
119 |
if filters and filters[-1] == "and":
|
120 |
filters.pop()
|
121 |
|
|
|
122 |
num_concurrent_calls = st.sidebar.number_input("Concurrent Calls:", min_value=1, max_value=10, value=5, step=1)
|
123 |
data_section = st.sidebar.text_area("Paste Data:")
|
124 |
paste_data = st.sidebar.button("Paste Data")
|
125 |
add_row = st.sidebar.button("Add Row")
|
126 |
reset = st.sidebar.button("Reset")
|
127 |
|
|
|
128 |
if paste_data:
|
129 |
pasted_urls = process_pasted_data(data_section)
|
130 |
for i, url in enumerate(pasted_urls):
|
|
|
138 |
if reset:
|
139 |
st.session_state.clear()
|
140 |
|
|
|
|
|
|
|
|
|
|
|
|
|
141 |
row_count = st.session_state.get("row_count", 1)
|
142 |
+
for i in range(row_count):
|
143 |
+
col1, col2 = st.columns([3, 1])
|
|
|
|
|
|
|
144 |
target_url_key = f"target_url_{i}"
|
145 |
+
with col1:
|
146 |
+
target_url = st.text_input(f"Enter the target URL {i + 1}", key=target_url_key)
|
147 |
+
df_key = f"df_{i}"
|
148 |
+
df = st.session_state.get(df_key)
|
149 |
+
with col2:
|
150 |
+
if df is not None:
|
151 |
+
csv = convert_df_to_csv(df)
|
152 |
+
st.download_button(
|
153 |
+
label=f"Download CSV for URL {i + 1}",
|
154 |
+
data=csv,
|
155 |
+
file_name=f'backlinks_{i + 1}.csv',
|
156 |
+
mime='text/csv',
|
157 |
+
)
|
158 |
+
elif df is None and st.sidebar.button("Generate All"):
|
159 |
+
st.error(f"Failed for URL {i + 1}: No data or error.")
|
160 |
+
|
161 |
+
generate_button = st.sidebar.button("Generate All")
|
162 |
+
|
163 |
+
if generate_button:
|
164 |
jobs = Queue()
|
165 |
results = Queue()
|
166 |
workers = [BacklinkWorker(jobs, results, api_login, api_key) for _ in range(num_concurrent_calls)]
|
|
|
167 |
for worker in workers:
|
168 |
worker.start()
|
|
|
169 |
for i in range(row_count):
|
170 |
target_url = st.session_state.get(f"target_url_{i}", "")
|
171 |
if target_url:
|
172 |
jobs.put((i, target_url, filters))
|
|
|
173 |
for _ in workers:
|
174 |
jobs.put(None)
|
|
|
175 |
for worker in workers:
|
176 |
worker.join()
|
|
|
177 |
while not results.empty():
|
178 |
url_id, df = results.get()
|
179 |
st.session_state[f"df_{url_id}"] = df
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|