Spaces:
Sleeping
Sleeping
File size: 6,380 Bytes
c5abee7 0a9ead7 c5abee7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 |
from langchain_community.document_loaders import WebBaseLoader
from langchain.prompts import ChatPromptTemplate
from langchain.output_parsers import ResponseSchema
from langchain.output_parsers import StructuredOutputParser
from langchain.prompts import PromptTemplate
from langchain.chat_models import ChatOpenAI
from langchain.chains import LLMChain
from dotenv import load_dotenv
import requests
import streamlit as st
import re
import openai
load_dotenv()
def is_shortened_url(url): # It is checking whether it is a shorten url or regular website url
try:
response = requests.head(url, allow_redirects=True)
final_url = response.url
if final_url != url:
return True
return False
except requests.exceptions.RequestException as e:
print("Error:", e)
return False
def expand_short_url(short_url): # It is converting shorten url to regular url
try:
response = requests.head(short_url, allow_redirects=True)
if response.status_code == 200:
return response.url
else:
print("Error: Short URL couldn't be expanded.")
return None
except requests.exceptions.RequestException as e:
print("Error:", e)
return None
def get_original_url(url):
if is_shortened_url(url):
return expand_short_url(url)
else:
return url
# This is the complete code where we are extracting content from the url using WebBaseLoader , using LLM to extract blog content only and then paraphrasing it
def paraphrased_post(url):
loader=WebBaseLoader([url],encoding='utf-8')
docs = loader.load()
template="""You are a helpful LinkedIn webscrapper. You are provided with a data , extract the content of the post only.
{docs}"""
prompt=PromptTemplate(template=template,input_variables=['docs'])
llm=ChatOpenAI(temperature=0)
chain=LLMChain(llm=llm,prompt=prompt)
result=chain.invoke({'docs':docs},return_only_outputs=True)
data=result['text']
template="""You are a helpful LinkedIn post paraphraser and plagiarism remover bot. You are provided with LinkedIn post content and your task is to paraphrase it and remove plagiarism .Return the output in the format with spaces or stickers if present.
{data}"""
prompt2=PromptTemplate(template=template,input_variables=['data'])
llm=ChatOpenAI(temperature=0)
chain2=LLMChain(llm=llm,prompt=prompt2)
result2=chain2({'data':data},return_only_outputs=True)
data2=extract_data(result2['text'])
keywords=data2['Keywords'][:3]
take_aways=data2['Take Aways'][:3]
highlights=data2['Highlights'][:3]
return result2['text'] ,keywords , take_aways, highlights
def extract_data(post_data):
keywords = ResponseSchema(name="Keywords",
description="These are the keywords extracted from LinkedIn post",type="list")
Take_aways = ResponseSchema(name="Take Aways",
description="These are the take aways extracted from LinkedIn post", type= "list")
Highlights=ResponseSchema(name="Highlights",
description="These are the highlights extracted from LinkedIn post", type= "list")
response_schema = [
keywords,
Take_aways,
Highlights
]
output_parser = StructuredOutputParser.from_response_schemas(response_schema)
format_instructions = output_parser.get_format_instructions()
template = """
You are a helpful keywords , take aways and highlights extractor from the post of LinkedIn Bot. Your task is to extract relevant keywords , take aways and highlights extractor.
From the following text message, extract the following information:
text message: {content}
{format_instructions}
"""
prompt_template = ChatPromptTemplate.from_template(template)
messages = prompt_template.format_messages(content=post_data, format_instructions=format_instructions)
llm = ChatOpenAI(temperature=0)
response = llm(messages)
output_dict= output_parser.parse(response.content)
return output_dict
def main():
st.title("Paraphrase LinkedIn Post")
# Initialize SessionState dictionary
session_state = st.session_state
if 'paraphrase' not in session_state:
session_state.paraphrase = ""
if 'keywords' not in session_state:
session_state.keywords = ""
if 'take_aways' not in session_state:
session_state.take_aways = ""
if 'highlights' not in session_state:
session_state.highlights = ""
# User input for URL
url = st.sidebar.text_input("Enter URL:", placeholder="Enter URL here...")
# Button to submit URL
if st.sidebar.button("Submit"):
if url:
original_url = get_original_url(url)
match = re.match(r"https?://(?:www\.)?linkedin\.com/(posts|feed|pulse)/.*", original_url) # checking domain and url page (means it should only be a post nothing else like login page or something else)
if match:
session_state.paraphrase, session_state.keywords, session_state.take_aways, session_state.highlights = paraphrased_post(url)
else:
st.sidebar.error("Put a valid LinkedIn post URL only")
paraphrase_text=st.text_area("Paraphrase:", value=session_state.paraphrase, height=400)
import pyperclip
if st.button('Copy'):
pyperclip.copy(paraphrase_text)
st.success('Text copied successfully!')
if st.sidebar.button("Show Keywords") and session_state.keywords:
st.write("Keywords:")
for i, statement in enumerate(session_state.keywords, start=1):
st.write(f"{i}. {statement}")
if st.sidebar.button("Show Take Aways") and session_state.take_aways:
st.write("Take Aways:")
for i, statement in enumerate(session_state.take_aways, start=1):
st.write(f"{i}. {statement}")
if st.sidebar.button("Show Highlights") and session_state.highlights:
st.write("Highlights:")
for i, statement in enumerate(session_state.highlights, start=1):
st.write(f"{i}. {statement}")
if __name__ == "__main__":
main()
|