updates

.gitignore

@@ -91,3 +91,4 @@ coverage.xml
 
 .idea 
 .vscode
+/data

data.dvc

@@ -1,5 +1,5 @@
 outs:
-- md5: 2ab20ac1b58df875a590b07d0e04eb5b.dir
-  path: data/raw
-  size: 1359144987
-  nfiles: 3
+- md5: 4088e0a288132d141c28bd020548d107.dir
+  path: data
+  size: 2720315628
+  nfiles: 6

src/data/process_data.py

@@ -3,12 +3,12 @@ import pandas as pd
 
 def process_data(split='train'):
     df = pd.read_csv('C:/Users/gbhat/Documents/GitHub/summarization/data/raw/{}.csv'.format(split))
-    df.rename(columns={"article": "input_text", "highlights": "output_text"})
-    print(df.shape())
+    df.columns = ['Unnamed: 0', 'input_text', 'output_text']
+    print(df.columns)
     df.to_csv('C:/Users/gbhat/Documents/GitHub/summarization/data/processed/{}.csv'.format(split))
 
 
-if __name__ == '__name__':
+if __name__ == '__main__':
     process_data(split='train')
     process_data(split='test')
     process_data(split='validation')