main.py

import os , json
from flask import Flask, render_template
import threading
import time 
from pydantic.v1.utils import unique_list
import requests

from langchain_core.tools import Tool
from langchain_google_community import GoogleSearchAPIWrapper, search
from langchain_community.tools import DuckDuckGoSearchResults

from langchain_community.utilities import DuckDuckGoSearchAPIWrapper



API_URL0 = "https://api-inference.huggingface.co/models/sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
API_URL1 = "https://api-inference.huggingface.co/models/sentence-transformers/all-mpnet-base-v2"
API_URL2 = "https://api-inference.huggingface.co/models/sentence-transformers/all-roberta-large-v1"
API_URL3 = "https://api-inference.huggingface.co/models/Snowflake/snowflake-arctic-embed-l-v2.0"
# API_URL4 = "https://api-inference.huggingface.co/models/sentence-transformers/multi-qa-MiniLM-L6-cos-v1"



search = GoogleSearchAPIWrapper()

bearer = "Bearer " + os.getenv('TOKEN')
headers = {"Authorization": bearer }
print("headers")
print(headers)

app = Flask(__name__)

@app.route('/app')
def server_app():
    llamafile = threading.Thread(target=threadserver)
    print('This /app will start the llamafile server on thread')
    llamafile.start()
    return 'llamafile.start()'

@app.route('/findsimilarity')
def server_one():     
    sourcesim = "Results" 
    s1 = "Results"    
    return render_template("similarity_1.html", sourcetxt = sourcesim, s1 = s1 , headertxt = bearer )


@app.route('/')
async def server_1():
    # TODO :: check html first then check similarity
    # TODO :: check parts of snipp to pass in the processing func 
    query_sentence = "capital city of the Philippines"
    duck_results = []
    all_results = []

    try:
        searchduck = DuckDuckGoSearchResults(output_format="list", num_results=20)
        duck_results = searchduck.invoke(query_sentence)
        print("type of duck")
        print(type(duck_results))
    except:
        print("An exception occurred")
        duck_results = []

    if type(duck_results) == list and len(duck_results) > 0 :
        all_results = duck_results   
    
    tool = Tool(
        name="google_search",
        description="Search Google for recent results.",
        func=search.run,
    )
    
    try:
        google_results = search.results( query_sentence , 10 )
        print("type(duck_results)") 
        print(type(duck_results))  
        print(type(all_results))  
    except:
        print("An exception occurred") 
    
    if type(google_results) == list  and len(google_results) > 0:
        all_results = all_results + google_results
        print("len of google and duck")
        print(len(all_results))
        print(len(google_results))
        print(len(duck_results))
    print("type of google")
    print(type(google_results))
    # print(all_results)
    all_snipps = []
    new_results = []
    # get the snippet put into list 
    split_query_words = query_sentence.split(); important_keywords = []; uppercased_keywords = [];
    for x in split_query_words:
        print(" x.isupper()  ")
        # print(x)
        # print( x[0].isupper()  )
        if x[0].isupper() == True :
            uppercased_keywords.append(x)
        if ( len(x) > 3 ) & ( x[0].isupper() == False ):             
            important_keywords.append(x)
    print("what is important and upper")
    print(important_keywords)
    print(uppercased_keywords)
    snipp_score = 0 
    capitalized_score = 0       
    for x in all_results:  
        snipp_score = 0 
        capitalized_score = 0  
        for words in important_keywords:
            # print("The important words " )
            # print(words)
            # print("x[snippet].find(words)")
            # print(x["snippet"].find(words))
            if x["snippet"].find(words) != -1 :
                # print("Found word")
                snipp_score = snipp_score + 1
        for words in uppercased_keywords:
            # print("The important words capitalized" )
            # print(words)
            if x["snippet"].find(words) != -1 :
                snipp_score = snipp_score + 1
                capitalized_score = capitalized_score + 1
            
        if ( snipp_score >= len(important_keywords) ) and ( ( capitalized_score <=  len(uppercased_keywords) and capitalized_score > 0 ) or ( len(uppercased_keywords) == 0 )  ):
            new_results.append(x)
            continue 
        if ( (snipp_score <= len(important_keywords) and snipp_score >= 2 ) and (len(important_keywords) <= 4) )  and ( (capitalized_score <= len(uppercased_keywords) and capitalized_score >= 1) or ( len(uppercased_keywords) == 0 )  ):
            new_results.append(x)
            continue 
        if ( ( snipp_score <= len(important_keywords) and snipp_score >= 4  ) and ( len(important_keywords) >= 5 and len(important_keywords) <= 7 )  ) and ( ( capitalized_score <=  len(uppercased_keywords) and capitalized_score > 0 ) or ( len(uppercased_keywords) == 0 )  ) :
            new_results.append(x)
            continue 
        else :
            # skip the result
            print("This is not added")
            # print(x["snippet"])
            # print("important keywords")
            # print(important_keywords)
            # print("capitalized_score")
            # print(capitalized_score)
            # print("snipp_score")
            # print(snipp_score)

    # print("these are new_results")
    # print("===============================")     

    # print(new_results)  
    
    # print("these are new_results")
    # print("===============================")     
    
    print( " len( new_results)  ")
    print( len( new_results)  ) 
    print("type of all_results")
    # TODO :: check html first then check similarity
    # TODO :: check parts of snipp to pass in the processing func     
    # TODO :: pull pages and split each html and count occurance of important keywords here & check snipp if snipp occurs between . and <p> its good not img
    
    n_results = {}
    iter_x = 0
    for x in new_results: 
        n_results[iter_x] = []
        print("x[snippet]")
        # print(x["snippet"])
        for y in (x["snippet"]).split('.') :            
            score = 0 ; cap_score  = 0 ;  
            for words in important_keywords :
                if y.find(words) != -1 :
                    # print(y)
                    # print(score)
                    score = score + 1 
            for words in uppercased_keywords :
                if y.find(words) != -1 :
                    # print(y)
                    # print(cap_score)
                    cap_score = cap_score + 1              
            if ( score == ( len(important_keywords) )  ) and ( cap_score >= ( len(uppercased_keywords) ) ):
                n_results[iter_x].append(y)            
            if ( score >= ( len(important_keywords)-1  )  ) or ( cap_score >=  len(uppercased_keywords) and (len(uppercased_keywords) > 0)  ):
                n_results[iter_x].append(y)
        iter_x = iter_x + 1
        # print("iterator") 
        # print(iter_x)

    print("n_results length")
    print(len(n_results))
    print("nresults")
    sentences_comparison = []
    iter_x = 0
    for y in n_results : 
        print("y")
        print(n_results[iter_x])
        print(y)
        # print(y) 
        for x in n_results[iter_x] : 
            sentences_comparison.append(x)
        iter_x = iter_x + 1
    
    print("sentences_comparison")
    print(sentences_comparison)

#     nresults={}
#     new_results loop
#        sentences loop
#           score = 0 ; cap_score  = 0
#           words loop 
#              if found score ++
#       
#          capitalized loop 
#              if found cap_score ++
#           if cap_score >= len words &&  if score >= len words 
#           
#          
#                 nresults[i].append(x)

                   





        



    # TODO :: check parts of snipp 
    # TODO :: check parts of snipp 
    # TODO :: check parts of snipp  
    
    payload = {  "inputs": {  "source_sentence": "Manila is the capital city of the Philippines",  "sentences": ["The current capital city, Manila, has been the countrys capital throughout most","Manila officially the City of Manila (Filipino: Lungsod ng Maynila),","Dis 4, 2024 — Manila, capital and chief city of the Philippines. The city is the centre ","Quezon City is the capital of the Philippines","Manila is the capital of the philippines","For sometime Manila has been the capital of of the Philippines" ,"What is the capital of Philippines","Manila is not the capital of the Phillipines","Quezon city was the capital of the Philippines, until President Ferdinand "] } , }
    response0 =  requests.post(API_URL0, headers=headers, json=payload)    
    response1 =  requests.post(API_URL1, headers=headers, json=payload)
    response2 =  requests.post(API_URL2, headers=headers, json=payload)
    response3 =  requests.post(API_URL3, headers=headers, json=payload)
    
    varcontinue_similarity = 0
    print("type( response0.json() )")
    print(type(  response0.json() ))
    print(type(  response1.json() ))
    print(type(  response2.json() ))
    print(type(  response3.json() ))
    
    if type(response0.json()) == list and type(response1.json()) == list and type(response2.json()) == list and type(response3.json()) == list : 
        similarity_scores =  response0.json() + response1.json() + response2.json() + response3.json()
        # If all list then pass to process func 
        sorted0 = sorted(response0.json() , reverse=True); sorted1 = sorted(response1.json() , reverse=True)
        sorted2 = sorted(response2.json() , reverse=True); sorted3 = sorted(response3.json() , reverse=True)
        varcontinue_similarity = 1
    else:
        similarity_scores = "There's an error in llm similarity search retrieval"
        return similarity_scores
    
    time.sleep(2) 
    result_processed = ""
    ## if response is all list  
    if varcontinue_similarity == 1 :
        # call processing with 10 google search result or 15 search results
        if len(all_results) == 10 :
            result_processed = process_similarity_15(sorted0, sorted1, sorted2, sorted3,response0.json(), response1.json(), response2.json(), response3.json()  )
        if len(all_results) > 10 :
            result_processed = process_similarity_15(sorted0, sorted1, sorted2, sorted3,response0.json(), response1.json(), response2.json(), response3.json()  )
    # return all_results
    return result_processed
    

        
def threadserver():
    print('hi')
    os.system(' ./mxbai-embed-large-v1-f16.llamafile --server --nobrowser')


 
def process_similarity_15(sorted0, sorted1, sorted2, sorted3, actualscore0, actualscore1, actualscore2, actualscore3):
    
    # print(similarity_scores)
    # print(type(similarity_scores))  
    print("length")
    # print(len(similarity_scores)) 
    key_index = 0 
    # copy + loop to get index  
    print("actual scores")
    print("actual scores")
    print(actualscore0)
    print(actualscore1)
    print(actualscore2)
    print(actualscore3)

    print("the sorted0-3")
    print("the sorted0-3")
    print(sorted0)
    print(sorted1)
    print(sorted2)
    print(sorted3)
    print("end the sorted0-3")
    # Get the index of the sorted list for resp_list0

    sorted0_with_index = []
    for x in sorted0:
        for y in actualscore0:
            if x == y:
                print("index of sorted0")
                print(actualscore0.index(y))
                if x > 0.90:
                    sorted0_with_index.append(actualscore0.index(y))
                    print("sorted_with_index")
                    print(sorted0_with_index)
    print("sorted0_with_index")    
    print(sorted0_with_index)    
    sorted1_with_index = []
    for x in sorted1:
        for y in actualscore1:
            if x == y:
                print("index of sorted1")
                print(actualscore1.index(y))
                if y > 0.90:
                    sorted1_with_index.append(actualscore1.index(y))
                    print("sorted_with_index")
                    print(sorted1_with_index)
    
    print("sorted1_with_index")
    print(sorted1_with_index)
    
    sorted2_with_index = []
    print("b4 for x in sorted2:")
    print("resp_list2:" + str(actualscore2))
    print("sorted:" + str(sorted2))
    for x in sorted2:
        for y in actualscore2:
            if x == y:
                print("index of sorted2")
                print(actualscore2.index(y))
                if y > 0.90:
                    sorted2_with_index.append(actualscore2.index(y))
                    print("sorted_with_index")
                    print(sorted2_with_index)
  
    print("sorted2_with_index")
    print(sorted2_with_index)
    sorted3_with_index = []
    print("b4 for x in sorted3:")
    print("resp_list3:" + str(actualscore3))
    for x in sorted3:
        for y in actualscore3:
            if x == y:
                print("index of sorted3")
                print(actualscore3.index(y))
                if y > 0.90:
                    sorted3_with_index.append(actualscore3.index(y))
                    print("sorted_with_index")
                    print(sorted3_with_index)
   
    print("sorted0-3_with_index")
    print("sorted0-3_with_index")
    print(sorted0_with_index)
    print(sorted1_with_index) 
    print(sorted2_with_index)
    print(sorted3_with_index)
    print("sorted0-3_with_index")
    

    # At this point the scores have been sorted also indexes are stored in lists 
    # At this point the scores have been sorted also indexes are stored in lists 
    this_unique_list = set( sorted0_with_index + sorted1_with_index + sorted2_with_index + sorted3_with_index )
    webgraph_list = [] 
    iterator_x = 0
    for x in sorted0_with_index:
        print(x)
        if ( x in sorted3_with_index and x in sorted1_with_index and x in sorted2_with_index ) :
            webgraph_list.append(x)
        if ( x in sorted1_with_index and x in sorted2_with_index ) or ( x in sorted3_with_index and x in sorted2_with_index ) or ( x in sorted1_with_index and x in sorted3_with_index ):
            webgraph_list.append(x)
        if (x in sorted1_with_index or x in sorted2_with_index or x in sorted3_with_index ) and actualscore0[iterator_x] > 0.96 :
            webgraph_list.append(x)
        iterator_x = iterator_x + 1
    
    print("webgraph_list0")
    print("webgraph_list0")
    print(webgraph_list)
    iterator_x = 0
    for x in sorted1_with_index:
        print(x)
        if x in sorted3_with_index and x in sorted0_with_index and x in sorted2_with_index :
            webgraph_list.append(x)
        if ( x in sorted0_with_index and x in sorted2_with_index ) or ( x in sorted3_with_index and x in sorted2_with_index ) or ( x in sorted0_with_index and x in sorted3_with_index ):
            webgraph_list.append(x)
        if (x in sorted0_with_index or x in sorted2_with_index or x in sorted3_with_index ) and actualscore1[iterator_x] > 0.96 :
            webgraph_list.append(x)
        iterator_x = iterator_x + 1
    
    print("webgraph_list1")
    print("webgraph_list1")
    print(webgraph_list)


    iterator_x = 0
    for x in sorted2_with_index:
        print(x)
        if x in sorted3_with_index and x in sorted0_with_index and x in sorted1_with_index :
            webgraph_list.append(x)
        if ( x in sorted0_with_index and x in sorted1_with_index ) or ( x in sorted3_with_index and x in sorted1_with_index ) or ( x in sorted0_with_index and x in sorted3_with_index ):
            webgraph_list.append(x)
        if (x in sorted0_with_index or x in sorted1_with_index or x in sorted3_with_index ) and actualscore2[iterator_x] > 0.96 :
            webgraph_list.append(x)
        iterator_x = iterator_x + 1
    
    print("webgraph_list2")
    print("webgraph_list2")
    print(webgraph_list)


    iterator_x = 0
    for x in sorted3_with_index:
        print(x)
        if x in sorted1_with_index and x in sorted0_with_index and x in sorted2_with_index :
            webgraph_list.append(x)
        if ( x in sorted0_with_index and x in sorted2_with_index ) or ( x in sorted1_with_index and x in sorted2_with_index ) or ( x in sorted0_with_index and x in sorted1_with_index ):
            webgraph_list.append(x)
        if (x in sorted0_with_index or x in sorted2_with_index or x in sorted1_with_index ) and actualscore3[iterator_x] > 0.96 :
            webgraph_list.append(x)
        iterator_x = iterator_x + 1
    
    print("webgraph_list3")
    print("webgraph_list3")
    print(webgraph_list)
    print("webgraph_list")
    print(webgraph_list)


    return str( list(set(webgraph_list ) ) ) 
 

if __name__ == '__main__':   
    app.run(host='0.0.0.0', port=8081)
    
    # server_app()