File size: 1,940 Bytes
9421cee |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
import pandas as pd
import re
# dictionary of state names to abbreviations
state_abbreviations = {
'Alabama': 'AL', 'Alaska': 'AK', 'Arizona': 'AZ', 'Arkansas': 'AR', 'California': 'CA',
'Colorado': 'CO', 'Connecticut': 'CT', 'Delaware': 'DE', 'Florida': 'FL', 'Georgia': 'GA',
'Hawaii': 'HI', 'Idaho': 'ID', 'Illinois': 'IL', 'Indiana': 'IN', 'Iowa': 'IA',
'Kansas': 'KS', 'Kentucky': 'KY', 'Louisiana': 'LA', 'Maine': 'ME', 'Maryland': 'MD',
'Massachusetts': 'MA', 'Michigan': 'MI', 'Minnesota': 'MN', 'Mississippi': 'MS', 'Missouri': 'MO',
'Montana': 'MT', 'Nebraska': 'NE', 'Nevada': 'NV', 'New Hampshire': 'NH', 'New Jersey': 'NJ',
'New Mexico': 'NM', 'New York': 'NY', 'North Carolina': 'NC', 'North Dakota': 'ND', 'Ohio': 'OH',
'Oklahoma': 'OK', 'Oregon': 'OR', 'Pennsylvania': 'PA', 'Rhode Island': 'RI', 'South Carolina': 'SC',
'South Dakota': 'SD', 'Tennessee': 'TN', 'Texas': 'TX', 'Utah': 'UT', 'Vermont': 'VT',
'Virginia': 'VA', 'Washington DC': 'DC', 'Washington': 'WA', 'West Virginia': 'WV', 'Wisconsin': 'WI', 'Wyoming': 'WY'
}
df = pd.read_csv('data/2019-climate-all.csv')
# remove duplicates
df.drop_duplicates(subset=['Username', 'Content'], inplace=True)
def get_state(location):
if not isinstance(location, str):
return None
# check for DC first
if re.search(r'\b(Washington DC|DC|D\.C)\b', location, re.IGNORECASE):
return 'Washington DC'
for state, abbrev in state_abbreviations.items():
pattern = rf'\b({re.escape(state)}|{re.escape(abbrev)})\b'
if re.search(pattern, location, re.IGNORECASE):
return state
return None
df['Filtered Location'] = df['User Location'].apply(get_state)
# filter rows where 'User Location (State)' is not blank
filtered_df = df[df['Filtered Location'].notna() & (df['Filtered Location'] != '')]
filtered_df.to_csv('data/2019-climate-usa-redo.csv', index=False)
|