app.py

# -*- coding: utf-8 -*-
"""
Created on Tue Feb 20 2023
@author: cyberandy
"""

# ---------------------------------------------------------------------------- #
# Imports
# ---------------------------------------------------------------------------- #
from io import StringIO  # for redirect_stdout
from functools import wraps  # for caching
import contextlib  # for redirect_stdout
import tldextract
import requests
import streamlit as st
import pandas as pd
import streamlit.components.v1 as components
import json
import os
from openai import OpenAI

# Unset any proxy environment variables that might be causing issues
proxy_vars = ['HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy']
for var in proxy_vars:
    if var in os.environ:
        del os.environ[var]
        
# ---------------------------------------------------------------------------- #
# App Config. & Styling
# ---------------------------------------------------------------------------- #


PAGE_CONFIG = {
    "page_title": "Structured Data Audit - a Free SEO Tool by WordLift",
    "page_icon": "img/fav-ico.png",
    "layout": "centered"
}


def local_css(file_name):
    with open(file_name) as f:
        st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True)


st.set_page_config(**PAGE_CONFIG)

local_css("style.css")


# ---------------------------------------------------------------------------- #
# Web Application
# ---------------------------------------------------------------------------- #
# st.title("🔥 Schema Audit 🔥")


# ---------------------------------------------------------------------------- #
# Sidebar
# ---------------------------------------------------------------------------- #
# st.sidebar.image("img/logo-wordlift.png", width=200)
# st.sidebar.info("Run the schema audit on any website to quickly get an overview of the available markup. \
#        Simply add the naked domain without 'www.' (eg. etsy.com or etsy.com/about) URL and click on ""ANALYZE"" to get the results.")
# st.sidebar.subheader("Configuration")

# ---------------------------------------------------------------------------- #
# Functions
# ---------------------------------------------------------------------------- #

# Set the API endpoint and the API key
API_ENDPOINT = "https://api2.woorank.com/reviews"
API_KEY = os.environ.get("woorank_api_key")
openai_api_key = os.environ.get("openai_api_key")

if not API_KEY:
    st.error("The API keys are not properly configured. Check your environment variables!")
elif not openai_api_key:
    st.error("The OpenAI API key is not properly configured. Check your environment variables!")
else:
    # Generate the report by calling the ChatGPT Turbo API and the WooRank API
    # First, let's create a simple PromptTemplate class since it's not imported
    class PromptTemplate:
        def __init__(self, template, input_variables):
            self.template = template
            self.input_variables = input_variables
        
        def format(self, **kwargs):
            return self.template.format(**kwargs)
    
    def analyze_data(_advice, _items, _topics, _issues, _technologies, openai_api_key):
        """
        Analyzes website data and generates a structured report using OpenAI's GPT model.
        
        Args:
            _advice (list): A list of strings, each string is a piece of advice
            _items (list): A list of items that are being analyzed
            _topics (list): A list of topics that the user is interested in
            _issues (list): A list of issues that the user has selected
            _technologies (list): A list of technologies that the user has selected
            openai_api_key (str): The OpenAI API key
        
        Returns:
            str: A JSON-formatted string containing the analysis report
        """
        try:
            # Create the system message for ChatGPT
            prefix_messages = [{
                "role": "system", 
                "content": '''You are an SEO expert specializing in structured data analysis. Your task is to create JSON-formatted reports about websites' structured data.
    
    Key requirements:
    1. Always format output as a valid JSON object
    2. Use the exact structure provided in the template
    3. Include HTML formatting (<i>, <b>, <u>) as specified
    4. Add relevant links to structured data (https://wordlift.io/blog/en/entity/structured-data/) and schema.org (https://wordlift.io/blog/en/entity/schema-org/) in the first section
    5. Keep responses concise but informative
    6. Ensure proper JSON escaping for quotes and special characters
    
    Remember: The output must be a single, valid JSON object that can be parsed without additional processing.'''
            }]
    
            # Initialize OpenAI client with basic configuration
            client = OpenAI(
                api_key=openai_api_key,
                base_url="https://api.openai.com/v1"
            )
            
            # Construct messages for the chat API
            messages = []
            messages.extend(prefix_messages)
    
            # Create the prompt template and run statement based on conditions
            if not _issues and len(_items) > 0:
                # Case 1: When there are NO issues but there ARE items
                template = """
                Create a JSON object based on the following data:
                1. {advice} and schema classes: {items}
                2. Entities found: {topics}
                3. Technologies: {technologies}
    
                Structure your response as a valid JSON object with this exact format:
                {{
                    "first": "Analysis of schema classes with classes marked in <i>italic</i>",
                    "second": "Description of entities marked in <b>bold</b>",
                    "third": "Description of technologies in <i>italic</i>"
                }}"""
                prompt = PromptTemplate(
                    template=template,
                    input_variables=["advice", "items", "topics", "technologies"]
                )
                run_statement = {
                    "advice": _advice,
                    "items": _items,
                    "topics": _topics,
                    "technologies": _technologies
                }
    
            elif not _items:
                # Case 2: When there are NO schema classes
                template = """
                Create a JSON object for a website with no schema classes, based on:
                1. Entities found: {topics}
                2. Technologies: {technologies}
    
                Structure your response as a valid JSON object with this exact format:
                {{
                    "first": "Notice about missing schema classes",
                    "second": "Description of entities marked in <b>bold</b>",
                    "third": "Description of technologies in <i>italic</i>"
                }}"""
                prompt = PromptTemplate(
                    template=template,
                    input_variables=["topics", "technologies"]
                )
                run_statement = {
                    "topics": _topics,
                    "technologies": _technologies
                }
    
            else:
                # Case 3: When there ARE issues
                template = """
                Create a JSON object based on the following data:
                1. {advice} and schema classes: {items}
                2. Markup issues: {issues}
                3. Entities found: {topics}
                4. Technologies: {technologies}
    
                Structure your response as a valid JSON object with this exact format:
                {{
                    "first": "Analysis of schema classes with classes marked in <i>italic</i>",
                    "second": "Description of issues marked in <u>underline</u>",
                    "third": "Description of entities marked in <b>bold</b>",
                    "fourth": "Description of technologies in <i>italic</i>"
                }}"""
                prompt = PromptTemplate(
                    template=template,
                    input_variables=["advice", "items", "topics", "issues", "technologies"]
                )
                run_statement = {
                    "advice": _advice,
                    "items": _items,
                    "topics": _topics,
                    "issues": _issues,
                    "technologies": _technologies
                }
            
            # Format the prompt and add it to messages
            user_message = prompt.format(**run_statement)
            messages.append({"role": "user", "content": user_message})
    
            # Make the API call with better error handling
            try:
                response = client.chat.completions.create(
                    model="gpt-4",
                    messages=messages,
                    temperature=0.7,
                    max_tokens=1500
                )
                if hasattr(response.choices[0].message, 'content'):
                    out = response.choices[0].message.content
                else:
                    out = "Error: No content in response"
            except Exception as e:
                error_msg = str(e)
                print(f"OpenAI API Error: {error_msg}")
                if "proxies" in error_msg:
                    out = "Error: Proxy configuration issue. Please check your environment settings."
                else:
                    out = f"Sorry, there was an error with the OpenAI API: {error_msg}"
    
            return out
        
        except Exception as e:
            error_message = f"An unexpected error occurred: {str(e)}"
            print(error_message)
            return error_message
    
    # Call WooRank API to get the data (cached)
    @st.cache_data
    def get_woorank_data(url):
        """
        It takes a URL as input, and returns a dictionary of the data from the Woorank API

        :param url: The URL of the website you want to get data for
        """
        # Extract the domain from the URL

        extracted = tldextract.extract(url)
        url = f"{extracted.domain}.{extracted.suffix}"

        # Build the API URL
        api_url = f"{API_ENDPOINT}?url={url}"

        # Set the API key in the headers
        headers = {"x-api-key": API_KEY,
                "Accept": "application/json"}

        # Call the API using HTTP GET and parse the JSON response to extract what we need
        response = requests.get(api_url, headers=headers)
        data = response.json()
        result = data.get("criteria", {}).get("schema_org", {})
        advice = result.get("advice", {})
        items = result.get("data", {}).get("counts", {})
        issues = result.get("data", {}).get("issues", {})
        topics_raw = data.get("criteria", {}).get("topics", {}).get("data", {})
        technologies_raw = data.get("criteria", {}).get(
            "technologies", {}).get("data", {}).get("technologies", {})

        # extract the unique English labels into a list
        topics = list(
            set([label for item in topics_raw for label in item['dbpediaLabelsEn']]))

        # extract the technologies that are related to seo and search-engines
        technologies = []
        for item in technologies_raw:
            if "seo" in item["categories"] or "search-engines" in item["categories"]:
                technologies.append(item["app"])

        # Return now all the items we need
        return result, advice, items, issues, topics, technologies

    # Here capture the output of the function and write it to the Streamlit app for debugging purposes
    def capture_output(func):
        """Capture output from running a function and write using streamlit."""

        @wraps(func)
        def wrapper(*args, **kwargs):
            # Redirect output to string buffers
            stdout, stderr = StringIO(), StringIO()
            try:
                with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
                    return func(*args, **kwargs)
            except Exception as err:
                print(f"Failure while executing: {err}")
            finally:
                if _stdout := stdout.getvalue():
                    print("Execution stdout:")
                    print(_stdout)
                if _stderr := stderr.getvalue():
                    print("Execution stderr:")
                    print(_stderr)

        return wrapper

    # ---------------------------------------------------------------------------- #
    # Main Function
    # ---------------------------------------------------------------------------- #
    def main():

        # Set up the Streamlit app
        # Adding the input for the URL
        url = st.text_input("Enter a URL to analyze")

        if st.button("RUN THE STRUCTURED DATA AUDIT"):
            # Call the Woorank API
            schema_org, advice, items, issues, topics, technologies = get_woorank_data(
                url)
            if not advice:
                st.warning("Whoops, sorry, our bot didn't find any data. It might be that the URL is not accessible (a firewall is in place, for example), or the website does not have structured data.", icon="⚠️")
            else:
                msg = analyze_data(advice, items, topics, issues, technologies, openai_api_key)
            # Display the results when the button is clicked and the data is available
                if schema_org and msg:
                    st.write("##### Your Findings 📈")
                    try:
                        data_out = json.loads(msg)
                        # here is the first block of text with the advice
                        first_block_text = data_out['first']
                        # here is the second block of text (opportunities if there are no issues, issues if there are)
                        second_block_text = data_out['second']

                        # here we create the HTML string for the first block of text (advice)
                        htmlstr1 = f"""<div class="success">{first_block_text}</div>"""
                        st.markdown(htmlstr1, unsafe_allow_html=True)
                        # adding a disclosure message
                        st.markdown(
                            """<div class="disclosure">*These findings are based on the analysis of your website as seen from the "eyes" of a crawler.</div>""", unsafe_allow_html=True)

                        # if there are no issues, we only have three blocks of text (advice, opportunities, technologies)
                        if not issues:
                            # here we get the third block of text with the technologies
                            third_block_text = data_out['third']
                            # here we create the HTML string for the second block of text (opportunities)
                            htmlstr2 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{second_block_text}</p>"""
                            st.markdown(htmlstr2, unsafe_allow_html=True)
                            # here we create the HTML string for the third block of text (technologies)
                            htmlstr3 = f"""<p class="technology">👩🏽‍💻 <b>Technologies</b></br>{third_block_text}</p>"""
                            st.markdown(htmlstr3, unsafe_allow_html=True)
                        # if there are issues, we have four blocks of text (advice, issues, opportunities, technologies)
                        else:
                            # here we get the third block of text with the opportunities
                            third_block_text = data_out['third']
                            # here we get the fourth block of text with the technologies
                            fourth_block_text = data_out['fourth']
                            # here we create the HTML string for the second block of text (issues)
                            htmlstr2 = f"""<p class="warning">⚠️ <b>Warnings</b></br>{second_block_text}</p>"""
                            st.markdown(htmlstr2, unsafe_allow_html=True)
                            # here we create the HTML string for the third block of text (opportunities)
                            htmlstr3 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{third_block_text}</p>"""
                            st.markdown(htmlstr3, unsafe_allow_html=True)
                            # here we create the HTML string for the fourth block of text (technologies)
                            htmlstr4 = f"""<p class="technology">👩🏽‍💻 <b>Technologies</b></br>{fourth_block_text}</p>"""
                            st.markdown(htmlstr4, unsafe_allow_html=True)
                    except Exception as e:
                        st.warning(
                            "Sorry, something went wrong. Please try again later.", icon="⚠️")
                        # Adding debug info
                        stprint = capture_output(print)
                        stprint(e)
                        stprint(msg)

                    st.write("---")

                    # Adding an expandable section to display the full response
                    with st.expander("INSPECT THE REPORT"):
                        # st.write("#### Advice")
                        # st.markdown(advice, unsafe_allow_html=True)
                        st.write("##### Items")
                        st.write(items)
                        if not issues:
                            st.write("No issues found on the structured data")
                        else:
                            st.write("#### Issues")
                            st.write(issues)
                        st.write("##### Entities")
                        st.write(topics)
                        st.write("##### Technologies")
                        st.write(technologies)
                        st.write("##### Full response")
                        st.write(schema_org)
                # If the API call fails, display an error message
        else:
            if len(url) == 0:
                st.warning("Please enter a URL to analyze")
    if __name__ == "__main__":
        main()