app.py

# -*- coding: utf-8 -*-
"""
Created on Tue Feb 20 2023
@author: cyberandy
"""

# ---------------------------------------------------------------------------- #
# Imports
# ---------------------------------------------------------------------------- #
from io import StringIO  # for redirect_stdout
from functools import wraps  # for caching
import contextlib  # for redirect_stdout
import tldextract
import requests
import streamlit as st
import pandas as pd
import streamlit.components.v1 as components
import json
import os
from openai import OpenAI


# ---------------------------------------------------------------------------- #
# App Config. & Styling
# ---------------------------------------------------------------------------- #


PAGE_CONFIG = {
    "page_title": "Structured Data Audit - a Free SEO Tool by WordLift",
    "page_icon": "img/fav-ico.png",
    "layout": "centered"
}


def local_css(file_name):
    with open(file_name) as f:
        st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True)


st.set_page_config(**PAGE_CONFIG)

local_css("style.css")


# ---------------------------------------------------------------------------- #
# Web Application
# ---------------------------------------------------------------------------- #
# st.title("🔥 Schema Audit 🔥")


# ---------------------------------------------------------------------------- #
# Sidebar
# ---------------------------------------------------------------------------- #
# st.sidebar.image("img/logo-wordlift.png", width=200)
# st.sidebar.info("Run the schema audit on any website to quickly get an overview of the available markup. \
#        Simply add the naked domain without 'www.' (eg. etsy.com or etsy.com/about) URL and click on ""ANALYZE"" to get the results.")
# st.sidebar.subheader("Configuration")

# ---------------------------------------------------------------------------- #
# Functions
# ---------------------------------------------------------------------------- #

# Set the API endpoint and the API key
API_ENDPOINT = "https://api2.woorank.com/reviews"
API_KEY = os.environ.get("woorank_api_key")
openai_api_key = os.environ.get("openai_api_key")

if not API_KEY:
    st.error("The API keys are not properly configured. Check your environment variables!")
elif not openai_api_key:
    st.error("The OpenAI API key is not properly configured. Check your environment variables!")
else:
    # Generate the report by calling the ChatGPT Turbo API and the WooRank API
    def analyze_data(_advice, _items, _topics, _issues, _technologies, openai_api_key):
        """
        :param _advice: A list of strings, each string is a piece of advice
        :param _items: a list of items that are being analyzed
        :param _topics: a list of topics that the user is interested in
        :param _issues: a list of issues that the user has selected
        :param _technologies: A list of technologies that the user has selected
        """
        # Create the system message for ChatGPT Turbo
        prefix_messages = [{"role": "system", "content": '''You are a helpful and truthful SEO that is very good at analyzing websites with a specific focus on structured data. /n
                            You are able to provide a detailed report on the website's structured data and how to improve it. /n
                            ADD AS LEARN MORE LINKS FOR THE FIRST TEXT BLOCK LINKS TO structured data https://wordlift.io/blog/en/entity/structured-data/ and schema.org https://wordlift.io/blog/en/entity/schema-org/ TO PROVIDE ADDITIONAL HELP./n/n
                            YOU ARE WRITING THE REPORT IN HTML USING A TEMPLATE.'''}]

        client = OpenAI(api_key=openai_api_key)
        
        # Construct messages for the chat API
        messages = []
        messages.extend(prefix_messages)


        # Create the prompt template and the run statement when there are NOT issues
        if not _issues and len(_items) > 0:
            template = """
            First text block of the report./n
            Analyze the: {advice}, consider that the site features the following schema classes: {items}./n/n

            Second text block of the report./n
            The website's homepage also references the following entities: {topics} that could be used to improve the SEO of the website further./n/n

            Third text block of the report./n
            Describe, if available, IN A SINGLE SENTENCE the {technologies} that the site appears to be using and what they do./n/n

            THE OUTPUT MUST USE THE FOLLOWING TEMPLATE:/n
            "first": "First text block with schema classes in <i>italic</i>",
            "second": "Second text block with entities in <b>bold</b>",
            "third": "Third text block with technologies in <i>italic</i>"
            """
            prompt = PromptTemplate(template=template, input_variables=[
                                    "advice", "items", "topics", "technologies"])
            run_statement = {"advice": _advice, "items": _items,
                            "topics": _topics, "technologies": _technologies}

        # Create the prompt template and the run statement when there ARE NOT schema classes
        elif not _items:
            template = """
            First text block of the report./n
            The website homepage doesn't seem to feature any schema class./n/n

            Second text block of the report./n
            The website's homepage also references the following entities: {topics} that can be used to improve the SEO of the website./n/n

            Third text block of the report./n
            Describe, if available, IN A SINGLE SENTENCE the {technologies} that the site appears to be using and what they do./n/n

            THE OUTPUT MUST USE THE FOLLOWING TEMPLATE:/n
            "first": "First text block",
            "second": "Second text block with entities in <b>bold</b>"
            "third": "Third text block with technologies in <i>italic</i>"
            """
            prompt = PromptTemplate(template=template, input_variables=[
                                    "topics", "technologies"])
            run_statement = {"topics": _topics, "technologies": _technologies}

        # Create the prompt template and the run statement when there ARE issues
        else:
            template = """
            First text block of the report./n
            Analyze the: {advice}, consider that the site features the following schema classes: {items}./n/n

            Second text block of the report. /n
            Describe the following issues with the markup: {issues} and indicate how to fix them./n/n

            Third text block of the report./n
            The website's homepage also references the following entities: {topics} that could be used to improve the SEO of the website further./n/n

            Fourth text block of the report./n
            Describe, if available, IN A SINGLE SENTENCE the {technologies} that the site appears to be using and what they do./n/n

            THE OUTPUT MUST USE THE FOLLOWING TEMPLATE:/n
            "first": "First text block with schema classes in <i>italic</i>",
            "second": "Second text block with issues in <u>underline</u>",
            "third": "Third text block with entities in <b>bold</b>"
            "fourth": "Fourth text block with technologies in <i>italic</i>"
            """
            prompt = PromptTemplate(template=template, input_variables=[
                                    "advice", "items", "topics", "issues", "technologies"])
            run_statement = {"advice": _advice, "items": _items,
                            "topics": _topics, "issues": _issues, "technologies": _technologies}
        
        # Format the prompt
        user_message = prompt.format(**run_statement)
        messages.append({"role": "user", "content": user_message})

        # Make the API call
        try:
            response = client.chat.completions.create(
                model="gpt-4o",
                messages=messages
            )
            out = response.choices[0].message.content
        except Exception as e:
            out = f"Sorry, there was an error with the OpenAI API: {e}"



        return out

    # Call WooRank API to get the data (cached)
    @st.cache_data
    def get_woorank_data(url):
        """
        It takes a URL as input, and returns a dictionary of the data from the Woorank API

        :param url: The URL of the website you want to get data for
        """
        # Extract the domain from the URL

        extracted = tldextract.extract(url)
        url = f"{extracted.domain}.{extracted.suffix}"

        # Build the API URL
        api_url = f"{API_ENDPOINT}?url={url}"

        # Set the API key in the headers
        headers = {"x-api-key": API_KEY,
                "Accept": "application/json"}

        # Call the API using HTTP GET and parse the JSON response to extract what we need
        response = requests.get(api_url, headers=headers)
        data = response.json()
        result = data.get("criteria", {}).get("schema_org", {})
        advice = result.get("advice", {})
        items = result.get("data", {}).get("counts", {})
        issues = result.get("data", {}).get("issues", {})
        topics_raw = data.get("criteria", {}).get("topics", {}).get("data", {})
        technologies_raw = data.get("criteria", {}).get(
            "technologies", {}).get("data", {}).get("technologies", {})

        # extract the unique English labels into a list
        topics = list(
            set([label for item in topics_raw for label in item['dbpediaLabelsEn']]))

        # extract the technologies that are related to seo and search-engines
        technologies = []
        for item in technologies_raw:
            if "seo" in item["categories"] or "search-engines" in item["categories"]:
                technologies.append(item["app"])

        # Return now all the items we need
        return result, advice, items, issues, topics, technologies

    # Here capture the output of the function and write it to the Streamlit app for debugging purposes
    def capture_output(func):
        """Capture output from running a function and write using streamlit."""

        @wraps(func)
        def wrapper(*args, **kwargs):
            # Redirect output to string buffers
            stdout, stderr = StringIO(), StringIO()
            try:
                with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
                    return func(*args, **kwargs)
            except Exception as err:
                print(f"Failure while executing: {err}")
            finally:
                if _stdout := stdout.getvalue():
                    print("Execution stdout:")
                    print(_stdout)
                if _stderr := stderr.getvalue():
                    print("Execution stderr:")
                    print(_stderr)

        return wrapper

    # ---------------------------------------------------------------------------- #
    # Main Function
    # ---------------------------------------------------------------------------- #
    def main():

        # Set up the Streamlit app
        # Adding the input for the URL
        url = st.text_input("Enter a URL to analyze")

        if st.button("RUN THE STRUCTURED DATA AUDIT"):
            # Call the Woorank API
            schema_org, advice, items, issues, topics, technologies = get_woorank_data(
                url)
            if not advice:
                st.warning("Whoops, sorry, our bot didn't find any data. It might be that the URL is not accessible (a firewall is in place, for example), or the website does not have structured data.", icon="⚠️")
            else:
                msg = analyze_data(advice, items, topics, issues, technologies, openai_api_key)
            # Display the results when the button is clicked and the data is available
                if schema_org and msg:
                    st.write("##### Your Findings 📈")
                    try:
                        data_out = json.loads(msg)
                        # here is the first block of text with the advice
                        first_block_text = data_out['first']
                        # here is the second block of text (opportunities if there are no issues, issues if there are)
                        second_block_text = data_out['second']

                        # here we create the HTML string for the first block of text (advice)
                        htmlstr1 = f"""<div class="success">{first_block_text}</div>"""
                        st.markdown(htmlstr1, unsafe_allow_html=True)
                        # adding a disclosure message
                        st.markdown(
                            """<div class="disclosure">*These findings are based on the analysis of your website as seen from the "eyes" of a crawler.</div>""", unsafe_allow_html=True)

                        # if there are no issues, we only have three blocks of text (advice, opportunities, technologies)
                        if not issues:
                            # here we get the third block of text with the technologies
                            third_block_text = data_out['third']
                            # here we create the HTML string for the second block of text (opportunities)
                            htmlstr2 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{second_block_text}</p>"""
                            st.markdown(htmlstr2, unsafe_allow_html=True)
                            # here we create the HTML string for the third block of text (technologies)
                            htmlstr3 = f"""<p class="technology">👩🏽‍💻 <b>Technologies</b></br>{third_block_text}</p>"""
                            st.markdown(htmlstr3, unsafe_allow_html=True)
                        # if there are issues, we have four blocks of text (advice, issues, opportunities, technologies)
                        else:
                            # here we get the third block of text with the opportunities
                            third_block_text = data_out['third']
                            # here we get the fourth block of text with the technologies
                            fourth_block_text = data_out['fourth']
                            # here we create the HTML string for the second block of text (issues)
                            htmlstr2 = f"""<p class="warning">⚠️ <b>Warnings</b></br>{second_block_text}</p>"""
                            st.markdown(htmlstr2, unsafe_allow_html=True)
                            # here we create the HTML string for the third block of text (opportunities)
                            htmlstr3 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{third_block_text}</p>"""
                            st.markdown(htmlstr3, unsafe_allow_html=True)
                            # here we create the HTML string for the fourth block of text (technologies)
                            htmlstr4 = f"""<p class="technology">👩🏽‍💻 <b>Technologies</b></br>{fourth_block_text}</p>"""
                            st.markdown(htmlstr4, unsafe_allow_html=True)
                    except Exception as e:
                        st.warning(
                            "Sorry, something went wrong. Please try again later.", icon="⚠️")
                        # Adding debug info
                        stprint = capture_output(print)
                        stprint(e)
                        stprint(msg)

                    st.write("---")

                    # Adding an expandable section to display the full response
                    with st.expander("INSPECT THE REPORT"):
                        # st.write("#### Advice")
                        # st.markdown(advice, unsafe_allow_html=True)
                        st.write("##### Items")
                        st.write(items)
                        if not issues:
                            st.write("No issues found on the structured data")
                        else:
                            st.write("#### Issues")
                            st.write(issues)
                        st.write("##### Entities")
                        st.write(topics)
                        st.write("##### Technologies")
                        st.write(technologies)
                        st.write("##### Full response")
                        st.write(schema_org)
                # If the API call fails, display an error message
        else:
            if len(url) == 0:
                st.warning("Please enter a URL to analyze")
            # else:
            #    st.warning("Whoops, sorry, our bot didn't find any data. It might be that the URL is not accessible (a firewall is in place, for example), or the website does not have structured data.", icon="⚠️")
    if __name__ == "__main__":
        main()