app.py

# -*- coding: utf-8 -*-
"""
Created on Tue Feb 20 2023
@author: cyberandy
"""

# ---------------------------------------------------------------------------- #
# Imports
# ---------------------------------------------------------------------------- #
from io import StringIO  # for redirect_stdout
from functools import wraps  # for caching
import contextlib  # for redirect_stdout
import tldextract
import requests
import streamlit as st
import pandas as pd
import streamlit.components.v1 as components
import json
import os
from openai import OpenAI

# Unset any proxy environment variables that might be causing issues
proxy_vars = ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']
for var in proxy_vars:
    if var in os.environ:
        del os.environ[var]
        
# ---------------------------------------------------------------------------- #
# App Config. & Styling
# ---------------------------------------------------------------------------- #


PAGE_CONFIG = {
    "page_title": "Structured Data Audit - a Free SEO Tool by WordLift",
    "page_icon": "img/fav-ico.png",
    "layout": "centered"
}


def local_css(file_name):
    with open(file_name) as f:
        st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True)


st.set_page_config(**PAGE_CONFIG)

local_css("style.css")


# ---------------------------------------------------------------------------- #
# Web Application
# ---------------------------------------------------------------------------- #
# st.title("🔥 Schema Audit 🔥")


# ---------------------------------------------------------------------------- #
# Sidebar
# ---------------------------------------------------------------------------- #
# st.sidebar.image("img/logo-wordlift.png", width=200)
# st.sidebar.info("Run the schema audit on any website to quickly get an overview of the available markup. \
#        Simply add the naked domain without 'www.' (eg. etsy.com or etsy.com/about) URL and click on ""ANALYZE"" to get the results.")
# st.sidebar.subheader("Configuration")

# ---------------------------------------------------------------------------- #
# Functions
# ---------------------------------------------------------------------------- #

# Set the API endpoint and the API key
API_ENDPOINT = "https://api2.woorank.com/reviews"
API_KEY = os.environ.get("woorank_api_key")
openai_api_key = os.environ.get("openai_api_key")

if not API_KEY:
    st.error("The API keys are not properly configured. Check your environment variables!")
elif not openai_api_key:
    st.error("The OpenAI API key is not properly configured. Check your environment variables!")
else:
    # Generate the report by calling the ChatGPT Turbo API and the WooRank API
    # First, let's create a simple PromptTemplate class since it's not imported
    class PromptTemplate:
        def __init__(self, template, input_variables):
            self.template = template
            self.input_variables = input_variables
        
        def format(self, **kwargs):
            return self.template.format(**kwargs)
    
    def analyze_data(_advice, _items, _topics, _issues, _technologies, openai_api_key):
        """
        Analyzes website data and generates a structured report using OpenAI's GPT model.
        
        Args:
            _advice (list): A list of strings, each string is a piece of advice
            _items (list): A list of items that are being analyzed
            _topics (list): A list of topics that the user is interested in
            _issues (list): A list of issues that the user has selected
            _technologies (list): A list of technologies that the user has selected
            openai_api_key (str): The OpenAI API key
        
        Returns:
            str: A JSON-formatted string containing the analysis report
        """
        try:
            # Create the system message for ChatGPT
            prefix_messages = [{
                "role": "system", 
                "content": '''You are a helpful and truthful SEO that is very good at analyzing websites with a specific focus on structured data.
                            You are able to provide a detailed report on the website's structured data and how to improve it.
                            ADD AS LEARN MORE LINKS FOR THE FIRST TEXT BLOCK LINKS TO structured data https://wordlift.io/blog/en/entity/structured-data/ and schema.org https://wordlift.io/blog/en/entity/schema-org/ TO PROVIDE ADDITIONAL HELP.
                            YOU ARE WRITING THE REPORT IN HTML USING A TEMPLATE.'''
            }]
    
            # Initialize OpenAI client with basic configuration
            client = OpenAI(
                api_key=openai_api_key,
                base_url="https://api.openai.com/v1",
                timeout=60.0  # 60 seconds timeout
            )
            
            # Construct messages for the chat API
            messages = []
            messages.extend(prefix_messages)
    
            # Create the prompt template and run statement based on conditions
            if not _issues and len(_items) > 0:
                # Case 1: When there are NO issues but there ARE items
                template = """
                First text block of the report.
                Analyze the: {advice}, consider that the site features the following schema classes: {items}.
    
                Second text block of the report.
                The website's homepage also references the following entities: {topics} that could be used to improve the SEO of the website further.
    
                Third text block of the report.
                Describe, if available, IN A SINGLE SENTENCE the {technologies} that the site appears to be using and what they do.
    
                THE OUTPUT MUST USE THE FOLLOWING TEMPLATE:
                "first": "First text block with schema classes in <i>italic</i>",
                "second": "Second text block with entities in <b>bold</b>",
                "third": "Third text block with technologies in <i>italic</i>"
                """
                prompt = PromptTemplate(
                    template=template,
                    input_variables=["advice", "items", "topics", "technologies"]
                )
                run_statement = {
                    "advice": _advice,
                    "items": _items,
                    "topics": _topics,
                    "technologies": _technologies
                }
    
            elif not _items:
                # Case 2: When there are NO schema classes
                template = """
                First text block of the report.
                The website homepage doesn't seem to feature any schema class.
    
                Second text block of the report.
                The website's homepage also references the following entities: {topics} that can be used to improve the SEO of the website.
    
                Third text block of the report.
                Describe, if available, IN A SINGLE SENTENCE the {technologies} that the site appears to be using and what they do.
    
                THE OUTPUT MUST USE THE FOLLOWING TEMPLATE:
                "first": "First text block",
                "second": "Second text block with entities in <b>bold</b>",
                "third": "Third text block with technologies in <i>italic</i>"
                """
                prompt = PromptTemplate(
                    template=template,
                    input_variables=["topics", "technologies"]
                )
                run_statement = {
                    "topics": _topics,
                    "technologies": _technologies
                }
    
            else:
                # Case 3: When there ARE issues
                template = """
                First text block of the report.
                Analyze the: {advice}, consider that the site features the following schema classes: {items}.
    
                Second text block of the report.
                Describe the following issues with the markup: {issues} and indicate how to fix them.
    
                Third text block of the report.
                The website's homepage also references the following entities: {topics} that could be used to improve the SEO of the website further.
    
                Fourth text block of the report.
                Describe, if available, IN A SINGLE SENTENCE the {technologies} that the site appears to be using and what they do.
    
                THE OUTPUT MUST USE THE FOLLOWING TEMPLATE:
                "first": "First text block with schema classes in <i>italic</i>",
                "second": "Second text block with issues in <u>underline</u>",
                "third": "Third text block with entities in <b>bold</b>",
                "fourth": "Fourth text block with technologies in <i>italic</i>"
                """
                prompt = PromptTemplate(
                    template=template,
                    input_variables=["advice", "items", "topics", "issues", "technologies"]
                )
                run_statement = {
                    "advice": _advice,
                    "items": _items,
                    "topics": _topics,
                    "issues": _issues,
                    "technologies": _technologies
                }
            
            # Format the prompt and add it to messages
            user_message = prompt.format(**run_statement)
            messages.append({"role": "user", "content": user_message})
    
            # Make the API call with better error handling
            try:
                response = client.chat.completions.create(
                    model="gpt-4",
                    messages=messages,
                    temperature=0.7,
                    max_tokens=1500
                )
                if hasattr(response.choices[0].message, 'content'):
                    out = response.choices[0].message.content
                else:
                    out = "Error: No content in response"
            except Exception as e:
                error_msg = str(e)
                print(f"OpenAI API Error: {error_msg}")  # Log the error for debugging
                if "proxies" in error_msg:
                    # Handle proxy-related errors
                    out = "Error: Proxy configuration issue. Please check your environment settings."
                else:
                    out = f"Sorry, there was an error with the OpenAI API: {error_msg}"
    
            return out
        
        except Exception as e:
            error_message = f"An unexpected error occurred: {str(e)}"
            print(error_message)  # Log the error for debugging
            return error_message
    
    # Call WooRank API to get the data (cached)
    @st.cache_data
    def get_woorank_data(url):
        """
        It takes a URL as input, and returns a dictionary of the data from the Woorank API

        :param url: The URL of the website you want to get data for
        """
        # Extract the domain from the URL

        extracted = tldextract.extract(url)
        url = f"{extracted.domain}.{extracted.suffix}"

        # Build the API URL
        api_url = f"{API_ENDPOINT}?url={url}"

        # Set the API key in the headers
        headers = {"x-api-key": API_KEY,
                "Accept": "application/json"}

        # Call the API using HTTP GET and parse the JSON response to extract what we need
        response = requests.get(api_url, headers=headers)
        data = response.json()
        result = data.get("criteria", {}).get("schema_org", {})
        advice = result.get("advice", {})
        items = result.get("data", {}).get("counts", {})
        issues = result.get("data", {}).get("issues", {})
        topics_raw = data.get("criteria", {}).get("topics", {}).get("data", {})
        technologies_raw = data.get("criteria", {}).get(
            "technologies", {}).get("data", {}).get("technologies", {})

        # extract the unique English labels into a list
        topics = list(
            set([label for item in topics_raw for label in item['dbpediaLabelsEn']]))

        # extract the technologies that are related to seo and search-engines
        technologies = []
        for item in technologies_raw:
            if "seo" in item["categories"] or "search-engines" in item["categories"]:
                technologies.append(item["app"])

        # Return now all the items we need
        return result, advice, items, issues, topics, technologies

    # Here capture the output of the function and write it to the Streamlit app for debugging purposes
    def capture_output(func):
        """Capture output from running a function and write using streamlit."""

        @wraps(func)
        def wrapper(*args, **kwargs):
            # Redirect output to string buffers
            stdout, stderr = StringIO(), StringIO()
            try:
                with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
                    return func(*args, **kwargs)
            except Exception as err:
                print(f"Failure while executing: {err}")
            finally:
                if _stdout := stdout.getvalue():
                    print("Execution stdout:")
                    print(_stdout)
                if _stderr := stderr.getvalue():
                    print("Execution stderr:")
                    print(_stderr)

        return wrapper

    # ---------------------------------------------------------------------------- #
    # Main Function
    # ---------------------------------------------------------------------------- #
    def main():

        # Set up the Streamlit app
        # Adding the input for the URL
        url = st.text_input("Enter a URL to analyze")

        if st.button("RUN THE STRUCTURED DATA AUDIT"):
            # Call the Woorank API
            schema_org, advice, items, issues, topics, technologies = get_woorank_data(
                url)
            if not advice:
                st.warning("Whoops, sorry, our bot didn't find any data. It might be that the URL is not accessible (a firewall is in place, for example), or the website does not have structured data.", icon="⚠️")
            else:
                msg = analyze_data(advice, items, topics, issues, technologies, openai_api_key)
            # Display the results when the button is clicked and the data is available
                if schema_org and msg:
                    st.write("##### Your Findings 📈")
                    try:
                        data_out = json.loads(msg)
                        # here is the first block of text with the advice
                        first_block_text = data_out['first']
                        # here is the second block of text (opportunities if there are no issues, issues if there are)
                        second_block_text = data_out['second']

                        # here we create the HTML string for the first block of text (advice)
                        htmlstr1 = f"""<div class="success">{first_block_text}</div>"""
                        st.markdown(htmlstr1, unsafe_allow_html=True)
                        # adding a disclosure message
                        st.markdown(
                            """<div class="disclosure">*These findings are based on the analysis of your website as seen from the "eyes" of a crawler.</div>""", unsafe_allow_html=True)

                        # if there are no issues, we only have three blocks of text (advice, opportunities, technologies)
                        if not issues:
                            # here we get the third block of text with the technologies
                            third_block_text = data_out['third']
                            # here we create the HTML string for the second block of text (opportunities)
                            htmlstr2 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{second_block_text}</p>"""
                            st.markdown(htmlstr2, unsafe_allow_html=True)
                            # here we create the HTML string for the third block of text (technologies)
                            htmlstr3 = f"""<p class="technology">👩🏽‍💻 <b>Technologies</b></br>{third_block_text}</p>"""
                            st.markdown(htmlstr3, unsafe_allow_html=True)
                        # if there are issues, we have four blocks of text (advice, issues, opportunities, technologies)
                        else:
                            # here we get the third block of text with the opportunities
                            third_block_text = data_out['third']
                            # here we get the fourth block of text with the technologies
                            fourth_block_text = data_out['fourth']
                            # here we create the HTML string for the second block of text (issues)
                            htmlstr2 = f"""<p class="warning">⚠️ <b>Warnings</b></br>{second_block_text}</p>"""
                            st.markdown(htmlstr2, unsafe_allow_html=True)
                            # here we create the HTML string for the third block of text (opportunities)
                            htmlstr3 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{third_block_text}</p>"""
                            st.markdown(htmlstr3, unsafe_allow_html=True)
                            # here we create the HTML string for the fourth block of text (technologies)
                            htmlstr4 = f"""<p class="technology">👩🏽‍💻 <b>Technologies</b></br>{fourth_block_text}</p>"""
                            st.markdown(htmlstr4, unsafe_allow_html=True)
                    except Exception as e:
                        st.warning(
                            "Sorry, something went wrong. Please try again later.", icon="⚠️")
                        # Adding debug info
                        stprint = capture_output(print)
                        stprint(e)
                        stprint(msg)

                    st.write("---")

                    # Adding an expandable section to display the full response
                    with st.expander("INSPECT THE REPORT"):
                        # st.write("#### Advice")
                        # st.markdown(advice, unsafe_allow_html=True)
                        st.write("##### Items")
                        st.write(items)
                        if not issues:
                            st.write("No issues found on the structured data")
                        else:
                            st.write("#### Issues")
                            st.write(issues)
                        st.write("##### Entities")
                        st.write(topics)
                        st.write("##### Technologies")
                        st.write(technologies)
                        st.write("##### Full response")
                        st.write(schema_org)
                # If the API call fails, display an error message
        else:
            if len(url) == 0:
                st.warning("Please enter a URL to analyze")
    if __name__ == "__main__":
        main()