nolanzandi commited on
Commit
1647e02
·
verified ·
1 Parent(s): b909f1c

Improve date parsing to reduce errors

Browse files
Files changed (1) hide show
  1. data_sources/upload_file.py +8 -2
data_sources/upload_file.py CHANGED
@@ -4,6 +4,7 @@ import csv
4
  import json
5
  import time
6
  import os
 
7
  from utils import TEMP_DIR
8
 
9
  def is_file_done_saving(file_path):
@@ -63,8 +64,13 @@ def process_data_upload(data_file, session_hash):
63
  df.columns = df.columns.str.replace('/', '_')
64
 
65
  for column in df.columns:
66
- if "date" in column.lower() or "time" in column.lower():
67
- df[column] = pd.to_datetime(df[column])
 
 
 
 
 
68
  if df[column].dtype == 'object' and isinstance(df[column].iloc[0], list):
69
  df[column] = df[column].explode()
70
 
 
4
  import json
5
  import time
6
  import os
7
+ import re
8
  from utils import TEMP_DIR
9
 
10
  def is_file_done_saving(file_path):
 
64
  df.columns = df.columns.str.replace('/', '_')
65
 
66
  for column in df.columns:
67
+ if type(column) is str:
68
+ pattern = 'year|month|date|day|time'
69
+ if re.search(pattern, column.lower()):
70
+ try:
71
+ df[column] = pd.to_datetime(df[column], infer_datetime_format=True)
72
+ except:
73
+ pass
74
  if df[column].dtype == 'object' and isinstance(df[column].iloc[0], list):
75
  df[column] = df[column].explode()
76