|
|
import os |
|
|
import streamlit as st |
|
|
import requests |
|
|
from streamlit_lottie import st_lottie |
|
|
|
|
|
def main() -> None: |
|
|
|
|
|
|
|
|
|
|
|
def load_lottieurl(lottie_url:str): |
|
|
r = requests.get(url=lottie_url) |
|
|
return r.json() if r.status_code == 200 else None |
|
|
|
|
|
def fetch(url): |
|
|
try: |
|
|
result = requests.post(url=os.environ.get('scraper-api-endpoint'), json={'url': url}) |
|
|
return result.json() |
|
|
except Exception: |
|
|
return {} |
|
|
|
|
|
st.set_page_config(page_title="Article Scraper - Rahul Portfolio Project", page_icon=":spider:", layout="wide") |
|
|
|
|
|
lottie_animation = load_lottieurl(lottie_url="https://assets3.lottiefiles.com/private_files/lf30_UaWyEa.json") |
|
|
|
|
|
|
|
|
with st.container(): |
|
|
st.subheader("Article Scraper") |
|
|
st.title("A Digital News / Article Information Extraction Application") |
|
|
st.write("A portfolio project developed to showcase my ability in developing Information Extraction Services") |
|
|
st.write("This service can be utilised in the data collection / curation process of data science workflow") |
|
|
st.write("[My Website >](https://www.rahulnenavath.co.in/)") |
|
|
st.subheader(f'About Article Scraper API:') |
|
|
st.write( |
|
|
""" |
|
|
- Article scraper API is deployed on AWS Lambda using AWS ECR container deployment |
|
|
- CI/CD workflow is implemented via GitHub Actions to push the latest docker build to AWS ECR |
|
|
- Check out the API codebase on [my GitHub >](https://github.com/RahulNenavath/Article-Scraper) |
|
|
API Tech Stack: Python, Beautifulsoup, AWS Lambda, AWS ECR, Docker, GitHub Actions (CI/CD) |
|
|
""" |
|
|
) |
|
|
|
|
|
with st.container(): |
|
|
st.write("---") |
|
|
left_col, right_col = st.columns(2) |
|
|
|
|
|
with left_col: |
|
|
st.header("How it works?") |
|
|
st.write("##") |
|
|
st.write('**Input**: Article URL') |
|
|
st.write('**Output**: Extracted Article Information') |
|
|
st.write( |
|
|
""" |
|
|
**Working**: |
|
|
- Download the HTML content from the given Article URL |
|
|
- Makes use of BeautifulSoup and extracts content from different HTML tags and ClassNames |
|
|
- Arrange Information appropriately |
|
|
- Regex based text cleaning to remove characters like additional spaces, unicodes, tabs, and newline characters |
|
|
""" |
|
|
) |
|
|
st.warning(f'Note: Web scraping is highly dependent on the Article HTML structure. Hence one might have to further clean the scraped content') |
|
|
|
|
|
with right_col: |
|
|
st_lottie(lottie_animation, height=500) |
|
|
|
|
|
with st.form("my_form"): |
|
|
article_url = st.text_input("Article URL", value="", key="article_url") |
|
|
|
|
|
submitted = st.form_submit_button("Submit") |
|
|
|
|
|
if submitted: |
|
|
with st.spinner('Scraping Information ...'): |
|
|
data = fetch(url=article_url) |
|
|
|
|
|
if data: |
|
|
st.success("Request is Successful") |
|
|
content = data.get("scraped_content") |
|
|
st.write("---") |
|
|
st.subheader(f'Extracted Article Information') |
|
|
st.write(f"**Article Title:** {content.get('article_title')}") |
|
|
st.write(f"**Author:** {content.get('author')}") |
|
|
st.write(f"**Published Date:** {content.get('publish_date')}") |
|
|
st.write(f"**Description:** {content.get('description')}") |
|
|
st.write(f"**Content:** {content.get('article_content')}") |
|
|
st.write(f"**Article URL:** {content.get('article_url')}") |
|
|
st.write(f"**Canonical URL:** {content.get('canonical_url')}") |
|
|
st.write(f"**Publisher Name:** {content.get('publisher_name')}") |
|
|
st.write(f"**Article Image:** {content.get('image')}") |
|
|
st.write(f"**Article Keywords:** {content.get('keywords')}") |
|
|
st.write(f"**Video URL:** {content.get('video_url')}") |
|
|
st.write(f"**Audio URL:** {content.get('audio_url')}") |
|
|
else: |
|
|
st.error("Request Failed") |
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
main() |
