Create app.py

app.py

@@ -0,0 +1,332 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Feb 20 2023
+@author: cyberandy
+"""
+
+# ---------------------------------------------------------------------------- #
+# Imports
+# ---------------------------------------------------------------------------- #
+from io import StringIO  # for redirect_stdout
+from functools import wraps  # for caching
+import contextlib  # for redirect_stdout
+import tldextract
+from langchain import PromptTemplate, LLMChain
+from langchain.llms import OpenAIChat
+import requests
+import streamlit as st
+import pandas as pd
+import streamlit.components.v1 as components
+import json
+import os
+
+
+# ---------------------------------------------------------------------------- #
+# App Config. & Styling
+# ---------------------------------------------------------------------------- #
+
+
+PAGE_CONFIG = {
+    "page_title": "Structured Data Audit - a Free SEO Tool by WordLift",
+    "page_icon": "img/fav-ico.png",
+    "layout": "centered"
+}
+
+
+def local_css(file_name):
+    with open(file_name) as f:
+        st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True)
+
+
+st.set_page_config(**PAGE_CONFIG)
+
+local_css("style.css")
+
+
+# ---------------------------------------------------------------------------- #
+# Web Application
+# ---------------------------------------------------------------------------- #
+# st.title("🔥 Schema Audit 🔥")
+
+
+# ---------------------------------------------------------------------------- #
+# Sidebar
+# ---------------------------------------------------------------------------- #
+# st.sidebar.image("img/logo-wordlift.png", width=200)
+# st.sidebar.info("Run the schema audit on any website to quickly get an overview of the available markup. \
+#        Simply add the naked domain without 'www.' (eg. etsy.com or etsy.com/about) URL and click on ""ANALYZE"" to get the results.")
+# st.sidebar.subheader("Configuration")
+
+# ---------------------------------------------------------------------------- #
+# Functions
+# ---------------------------------------------------------------------------- #
+
+# Set the API endpoint and the API key
+API_ENDPOINT = "https://api2.woorank.com/reviews"
+API_KEY = st.secrets["woorank_api_key"]
+os.environ["OPENAI_API_KEY"] = st.secrets["openai_api_key"]
+
+# Generate the report by calling the ChatGPT Turbo API and the WooRank API
+
+
+def analyze_data(_advice, _items, _topics, _issues, _technologies):
+    """
+    :param _advice: A list of strings, each string is a piece of advice
+    :param _items: a list of items that are being analyzed
+    :param _topics: a list of topics that the user is interested in
+    :param _issues: a list of issues that the user has selected
+    :param _technologies: A list of technologies that the user has selected
+    """
+    # Create the system message for ChatGPT Turbo
+    prefix_messages = [{"role": "system", "content": '''You are a helpful and truthful SEO that is very good at analyzing websites with a specific focus on structured data. /n
+                        You are able to provide a detailed report on the website's structured data and how to improve it. /n
+                        ADD AS LEARN MORE LINKS FOR THE FIRST TEXT BLOCK LINKS TO structured data https://wordlift.io/blog/en/entity/structured-data/ and schema.org https://wordlift.io/blog/en/entity/schema-org/ TO PROVIDE ADDITIONAL HELP./n/n
+                        YOU ARE WRITING THE REPORT IN HTML USING A TEMPLATE.'''}]
+    llm = OpenAIChat(temperature=0, prefix_messages=prefix_messages)
+
+    # Create the prompt template and the run statement when there are NOT issues
+    if not _issues and len(_items) > 0:
+        template = """
+        First text block of the report./n
+        Analyze the: {advice}, consider that the site features the following schema classes: {items}./n/n
+
+        Second text block of the report./n
+        The website's homepage also references the following entities: {topics} that could be used to improve the SEO of the website further./n/n
+
+        Third text block of the report./n
+        Describe, if available, IN A SINGLE SENTENCE the {technologies} that the site appears to be using and what they do./n/n
+
+        THE OUTPUT MUST USE THE FOLLOWING TEMPLATE:/n
+        "first": "First text block with schema classes in <i>italic</i>",
+        "second": "Second text block with entities in <b>bold</b>",
+        "third": "Third text block with technologies in <i>italic</i>"
+        """
+        prompt = PromptTemplate(template=template, input_variables=[
+                                "advice", "items", "topics", "technologies"])
+        run_statement = {"advice": _advice, "items": _items,
+                         "topics": _topics, "technologies": _technologies}
+
+    # Create the prompt template and the run statement when there ARE NOT schema classes
+    elif not _items:
+        template = """
+        First text block of the report./n
+        The website homepage doesn't seem to feature any schema class./n/n
+
+        Second text block of the report./n
+        The website's homepage also references the following entities: {topics} that can be used to improve the SEO of the website./n/n
+
+        Third text block of the report./n
+        Describe, if available, IN A SINGLE SENTENCE the {technologies} that the site appears to be using and what they do./n/n
+
+        THE OUTPUT MUST USE THE FOLLOWING TEMPLATE:/n
+        "first": "First text block",
+        "second": "Second text block with entities in <b>bold</b>"
+        "third": "Third text block with technologies in <i>italic</i>"
+        """
+        prompt = PromptTemplate(template=template, input_variables=[
+                                "topics", "technologies"])
+        run_statement = {"topics": _topics, "technologies": _technologies}
+
+    # Create the prompt template and the run statement when there ARE issues
+    else:
+        template = """
+        First text block of the report./n
+        Analyze the: {advice}, consider that the site features the following schema classes: {items}./n/n
+
+        Second text block of the report. /n
+        Describe the following issues with the markup: {issues} and indicate how to fix them./n/n
+
+        Third text block of the report./n
+        The website's homepage also references the following entities: {topics} that could be used to improve the SEO of the website further./n/n
+
+        Fourth text block of the report./n
+        Describe, if available, IN A SINGLE SENTENCE the {technologies} that the site appears to be using and what they do./n/n
+
+        THE OUTPUT MUST USE THE FOLLOWING TEMPLATE:/n
+        "first": "First text block with schema classes in <i>italic</i>",
+        "second": "Second text block with issues in <u>underline</u>",
+        "third": "Third text block with entities in <b>bold</b>"
+        "fourth": "Fourth text block with technologies in <i>italic</i>"
+        """
+        prompt = PromptTemplate(template=template, input_variables=[
+                                "advice", "items", "topics", "issues", "technologies"])
+        run_statement = {"advice": _advice, "items": _items,
+                         "topics": _topics, "issues": _issues, "technologies": _technologies}
+
+    # Create the LLMChain
+    llm_chain = LLMChain(prompt=prompt, llm=llm)
+
+    # If there are no issues, remove the issues from the prompt
+    prompt_text = prompt.format(**run_statement)
+
+    # Run the LLMChain and return the output
+    out = llm_chain.run(**run_statement)
+
+    return out
+
+# Call WooRank API to get the data (cached)
+
+
+@st.cache_data
+def get_woorank_data(url):
+    """
+    It takes a URL as input, and returns a dictionary of the data from the Woorank API
+
+    :param url: The URL of the website you want to get data for
+    """
+    # Extract the domain from the URL
+
+    extracted = tldextract.extract(url)
+    url = f"{extracted.domain}.{extracted.suffix}"
+
+    # Build the API URL
+    api_url = f"{API_ENDPOINT}?url={url}"
+
+    # Set the API key in the headers
+    headers = {"x-api-key": "456b66d0ed4e3c9a2d7b559e40ce9034",
+               "Accept": "application/json"}
+
+    # Call the API using HTTP GET and parse the JSON response to extract what we need
+    response = requests.get(api_url, headers=headers)
+    data = response.json()
+    result = data.get("criteria", {}).get("schema_org", {})
+    advice = result.get("advice", {})
+    items = result.get("data", {}).get("counts", {})
+    issues = result.get("data", {}).get("issues", {})
+    topics_raw = data.get("criteria", {}).get("topics", {}).get("data", {})
+    technologies_raw = data.get("criteria", {}).get(
+        "technologies", {}).get("data", {}).get("technologies", {})
+
+    # extract the unique English labels into a list
+    topics = list(
+        set([label for item in topics_raw for label in item['dbpediaLabelsEn']]))
+
+    # extract the technologies that are related to seo and search-engines
+    technologies = []
+    for item in technologies_raw:
+        if "seo" in item["categories"] or "search-engines" in item["categories"]:
+            technologies.append(item["app"])
+
+    # Return now all the items we need
+    return result, advice, items, issues, topics, technologies
+
+# Here capture the output of the function and write it to the Streamlit app for debugging purposes
+
+
+def capture_output(func):
+    """Capture output from running a function and write using streamlit."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        # Redirect output to string buffers
+        stdout, stderr = StringIO(), StringIO()
+        try:
+            with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
+                return func(*args, **kwargs)
+        except Exception as err:
+            print(f"Failure while executing: {err}")
+        finally:
+            if _stdout := stdout.getvalue():
+                print("Execution stdout:")
+                print(_stdout)
+            if _stderr := stderr.getvalue():
+                print("Execution stderr:")
+                print(_stderr)
+
+    return wrapper
+
+# ---------------------------------------------------------------------------- #
+# Main Function
+# ---------------------------------------------------------------------------- #
+
+
+def main():
+
+    # Set up the Streamlit app
+    # Adding the input for the URL
+    url = st.text_input("Enter a URL to analyze")
+
+    if st.button("RUN THE STRUCTURED DATA AUDIT"):
+        # Call the Woorank API
+        schema_org, advice, items, issues, topics, technologies = get_woorank_data(
+            url)
+        msg = analyze_data(advice, items, topics, issues, technologies)
+        # Display the results when the button is clicked and the data is available
+        if schema_org:
+            st.write("##### Your Findings 📈")
+            try:
+                data_out = json.loads(msg)
+                # here is the first block of text with the advice
+                first_block_text = data_out['first']
+                # here is the second block of text (opportunities if there are no issues, issues if there are)
+                second_block_text = data_out['second']
+
+                # here we create the HTML string for the first block of text (advice)
+                htmlstr1 = f"""<div class="success">{first_block_text}</div>"""
+                st.markdown(htmlstr1, unsafe_allow_html=True)
+                # adding a disclosure message
+                st.markdown(
+                    """<div class="disclosure">*These findings are based on the analysis of your website as seen from the "eyes" of a crawler.</div>""", unsafe_allow_html=True)
+
+                # if there are no issues, we only have three blocks of text (advice, opportunities, technologies)
+                if not issues:
+                    # here we get the third block of text with the technologies
+                    third_block_text = data_out['third']
+                    # here we create the HTML string for the second block of text (opportunities)
+                    htmlstr2 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{second_block_text}</p>"""
+                    st.markdown(htmlstr2, unsafe_allow_html=True)
+                    # here we create the HTML string for the third block of text (technologies)
+                    htmlstr3 = f"""<p class="technology">👩🏽‍💻 <b>Technologies</b></br>{third_block_text}</p>"""
+                    st.markdown(htmlstr3, unsafe_allow_html=True)
+                # if there are issues, we have four blocks of text (advice, issues, opportunities, technologies)
+                else:
+                    # here we get the third block of text with the opportunities
+                    third_block_text = data_out['third']
+                    # here we get the fourth block of text with the technologies
+                    fourth_block_text = data_out['fourth']
+                    # here we create the HTML string for the second block of text (issues)
+                    htmlstr2 = f"""<p class="warning">⚠️ <b>Warnings</b></br>{second_block_text}</p>"""
+                    st.markdown(htmlstr2, unsafe_allow_html=True)
+                    # here we create the HTML string for the third block of text (opportunities)
+                    htmlstr3 = f"""<p class="opportunity">ℹ️ <b>Opportunities</b></br>{third_block_text}</p>"""
+                    st.markdown(htmlstr3, unsafe_allow_html=True)
+                    # here we create the HTML string for the fourth block of text (technologies)
+                    htmlstr4 = f"""<p class="technology">👩🏽‍💻 <b>Technologies</b></br>{fourth_block_text}</p>"""
+                    st.markdown(htmlstr4, unsafe_allow_html=True)
+            except Exception as e:
+                st.warning(
+                    "Sorry, something went wrong. Please try again later.", icon="⚠️")
+                # Adding debug info
+                stprint = capture_output(print)
+                stprint(e)
+                stprint(msg)
+
+            st.write("---")
+
+            # Adding an expandable section to display the full response
+            with st.expander("INSPECT THE REPORT"):
+                # st.write("#### Advice")
+                # st.markdown(advice, unsafe_allow_html=True)
+                st.write("##### Items")
+                st.write(items)
+                if not issues:
+                    st.write("No issues found on the structured data")
+                else:
+                    st.write("#### Issues")
+                    st.write(issues)
+                st.write("##### Entities")
+                st.write(topics)
+                st.write("##### Technologies")
+                st.write(technologies)
+                st.write("##### Full response")
+                st.write(schema_org)
+        # If the API call fails, display an error message
+        else:
+            if len(url) == 0:
+                st.warning("Please enter a URL to analyze")
+            else:
+                st.warning("Whoops, sorry, our bot didn't find any data. It might be that the URL is not accessible (a firewall is in place, for example), or the website does not have structured data.", icon="⚠️")
+
+
+if __name__ == "__main__":
+    main()