Update main.py

main.py

@@ -1,15 +1,14 @@
 from fastapi import FastAPI
 import uvicorn
 
+from geopy.extra.rate_limiter import RateLimiter
+from geopy.geocoders import Nominatim
 import pandas as pd
 import numpy as np
-import requests
-from urllib.parse import urlparse, quote
-import re
-from bs4 import BeautifulSoup
-import time
-from joblib import Parallel, delayed
-from nltk import ngrams
+import pickle
+import rasterio
+import h5py
+from skimage.morphology import disk 
 
 app = FastAPI()
 
@@ -18,124 +17,101 @@ app = FastAPI()
 #Root endpoints
 @app.get("/")
 def root():
-    return {"API": "Google Address Scrap"}
+    return {"API": "Hail Docker Data"}
 
 
+def geocode_address(address):
 
-def normalize_string(string):
-    normalized_string = string.lower()
-    normalized_string = re.sub(r'[^\w\s]', '', normalized_string)
-    
-    return normalized_string
-
-
-def jaccard_similarity(string1, string2,n = 2, normalize=True):
     try:
-        if normalize:
-           string1,string2= normalize_string(string1),normalize_string(string2)
-           
-        grams1 = set(ngrams(string1, n))
-        grams2 = set(ngrams(string2, n))
-        similarity = len(grams1.intersection(grams2)) / len(grams1.union(grams2))
+        address2 = address.replace(' ', '+').replace(',', '%2C')
+        df = pd.read_json(
+            f'https://geocoding.geo.census.gov/geocoder/locations/onelineaddress?address={address2}&benchmark=2020&format=json')
+        results = df.iloc[:1, 0][0][0]['coordinates']
+        lat, lon = results['y'], results['x']
     except:
-        similarity=0
+        geolocator = Nominatim(user_agent='GTA Lookup')
+        geocode = RateLimiter(geolocator.geocode, min_delay_seconds=2)
+        location = geolocator.geocode(address)
+        lat, lon = location.latitude, location.longitude
         
-    if string2=='did not extract address':
-        similarity=0
-        
-    return similarity
-
-def jaccard_sim_split_word_number(string1,string2):
-    numbers1 = ' '.join(re.findall(r'\d+', string1))
-    words1 = ' '.join(re.findall(r'\b[A-Za-z]+\b', string1))
-    
-    numbers2 = ' '.join(re.findall(r'\d+', string2))
-    words2 = ' '.join(re.findall(r'\b[A-Za-z]+\b', string2))   
-    
-    number_similarity=jaccard_similarity(numbers1,numbers2)
-    words_similarity=jaccard_similarity(words1,words2)
-    return (number_similarity+words_similarity)/2
-
-def extract_website_domain(url):
-    parsed_url = urlparse(url)
-    return parsed_url.netloc
+    return lat, lon
 
 
-def google_address(address):     
+def get_hail_data(address, start_date,end_date,radius_miles):
 
-    search_query = quote(address)
-    url=f'https://www.google.com/search?q={search_query}'
-    response = requests.get(url)
-    soup = BeautifulSoup(response.content, "html.parser")
-    
-    texts_links = []
-    for link in soup.find_all("a"):
-        t,l=link.get_text(), link.get("href")
-        if (l[:11]=='/url?q=http') and (len(t)>20 ):
-            texts_links.append((t,l))
+    start_date = pd.Timestamp(str(start_date)).strftime('%Y%m%d')
+    end_date = pd.Timestamp(str(end_date)).strftime('%Y%m%d')
+    date_years = pd.date_range(start=start_date, end=end_date, freq='M')
+    years = list(set([d.year for d in date_years]))
     
-    text = soup.get_text()
     
-    texts_links_des=[]
-    for i,t_l in enumerate(texts_links):
-        start=text.find(texts_links[i][0][:50])
-        try:
-            end=text.find(texts_links[i+1][0][:50])
-        except:
-            end=text.find('Related searches')
-            
-        description=text[start:end]
-        texts_links_des.append((t_l[0],t_l[1],description))
+    if len(years)==0:
+        years=[pd.Timestamp(start_date).year]
     
-    df=pd.DataFrame(texts_links_des,columns=['Title','Link','Description'])
-    df['Description']=df['Description'].bfill()
-    df['Address Output']=df['Title'].str.extract(r'(.+? \d{5})').fillna("**DID NOT EXTRACT ADDRESS**")
+    # Geocode Address
+    lat, lon= geocode_address(address)
     
-    df['Link']=[i[7:i.find('&sa=')] for i in df['Link']]
-    df['Website'] = df['Link'].apply(extract_website_domain)
+    # Convert Lat Lon to row & col on Array
+    transform = pickle.load(open('Data/transform_mrms.pkl', 'rb')) 
+    row, col = rasterio.transform.rowcol(transform, lon, lat)
     
-    df['Square Footage']=df['Description'].str.extract(r"((\d+) Square Feet|(\d+) sq. ft.|(\d+) sqft|(\d+) Sq. Ft.|(\d+) sq|(\d+(?:,\d+)?) Sq\. Ft\.|(\d+(?:,\d+)?) sq)")[0]
-    try:
-        df['Square Footage']=df['Square Footage'].replace({',':''},regex=True).str.replace(r'\D', '')
-    except:
-        pass
-    df['Beds']=df['Description'].replace({'-':' ','total':''},regex=True).str.extract(r"(\d+) bed")
+    files = [
+        'Data/2023_hail.h5',
+        'Data/2022_hail.h5',
+        'Data/2021_hail.h5',
+        'Data/2020_hail.h5'
+        ]
     
+    files_choosen=[i for i in files if any(i for j in years if str(j) in i)]
     
-    df['Baths']=df['Description'].replace({'-':' ','total':''},regex=True).str.extract(r"((\d+) bath|(\d+(?:\.\d+)?) bath)")[0]
-    df['Baths']=df['Baths'].str.extract(r'([\d.]+)').astype(float)
+    # Query and Collect H5 Data  
+    all_data=[]
+    all_dates=[]
+    for file in files_choosen:
+        with h5py.File(file, 'r') as f:
+            # Get Dates from H5
+            dates = f['dates'][:]
+            date_idx=np.where((dates>=int(start_date)) & (dates<=int(end_date)) )[0]
+            
+            # Select Data by Date and Radius
+            dates=dates[date_idx]
+            data = f['hail'][date_idx, row-radius_miles:row +
+                              radius_miles+1, col-radius_miles:col+radius_miles+1]
+        
+            all_data.append(data)
+            all_dates.append(dates)
     
-    df['Year Built']=df['Description'].str.extract(r"built in (\d{4})")
+    data_all=np.vstack(all_data)
+    dates_all=np.concatenate(all_dates)
     
-    df['Match Percent']=[jaccard_sim_split_word_number(address,i)*100 for i in df['Address Output']]
-    df['Google Search Result']=[*range(1,df.shape[0]+1)]
-   
-    df.insert(0,'Address Input',address)
+    # Convert to Inches
+    data_mat = np.where(data_all < 0, 0, data_all)*0.0393701 
     
-    return df
-
+    # Get Radius of Data
+    disk_mask = np.where(disk(radius_miles)==1,True, False)
+    data_mat = np.where(disk_mask, data_mat, -1)
     
-def catch_errors(addresses):
-    try: 
-        return google_address(addresses)
-    except:
-        return pd.DataFrame({'Address Input':[addresses]})
-
-
-def process_multiple_address(addresses):
-    results=Parallel(n_jobs=32, prefer="threads")(delayed(catch_errors)(i) for i in addresses)
-    return results
+    # Find Max of Data
+    data_max = np.max(data_mat, axis=(1, 2)).round(3)
     
+    # Process to DataFrame
+    date_range_days = pd.date_range(start_date,end_date)
+    df_data=pd.DataFrame({'Date':dates_all,
+                          'Hail_max':data_max})
+    df_data['Date']=pd.to_datetime(df_data['Date'],format='%Y%m%d')
+    df_data=df_data.set_index('Date')
     
-@app.get('/Google_Address_Scrap')
-async def predict(address_input: str):
+    df_data = df_data.reindex(date_range_days, fill_value=0).reset_index().rename(columns={'index': 'Date'})
+    df_data['Date']=df_data['Date'].dt.strftime('%Y-%m-%d')
+    return df_data
+
     
-    address_input_split = address_input.split(';')
-    results = process_multiple_address(address_input_split)
-    results = pd.concat(results).reset_index(drop=1)
-    prediction = results[['Address Input', 'Address Output', 'Match Percent', 'Website', 'Square Footage', 'Beds', 'Baths', 'Year Built',
-                       'Link', 'Google Search Result', 'Description']]
-    return prediction.to_json()
+@app.get('/Hail_Docker_Data')
+async def predict(address, start_date,end_date, radius_miles):
     
-
-
+    try:
+        results = get_hail_data(address, start_date,end_date, radius_miles)
+    except:
+        results=pd.DataFrame({'Date':['error'],'df_data':['error']})
+        
+    return results.to_json()