updates

src/data/make_dataset.py

@@ -8,7 +8,7 @@ def make_dataset(dataset='cnn_dailymail', split='train'):
     df = pd.DataFrame()
     df['article'] = dataset['article']
     df['highlights'] = dataset['highlights']
-    df.to_csv('summarization/data/raw/{}.csv'.format(split))
+    df.to_csv('C:/Users/gbhat/Documents/GitHub/summarization/data/raw/{}.csv'.format(split))
 
 
 if __name__ == '__main__':

src/data/process_data.py

@@ -2,11 +2,10 @@ import pandas as pd
 
 
 def process_data(split='train'):
-    df = pd.DataFrame()
-    dataset = pd.load_csv('summarization/data/raw/{}.csv'.format(split))
-    df['article'] = dataset['article']
-    df['highlights'] = dataset['highlights']
-    df.to_csv('summarization/data/processed/{}.csv'.format(split))
+    df = pd.read_csv('C:/Users/gbhat/Documents/GitHub/summarization/data/raw/{}.csv'.format(split))
+    df.rename(columns={"article": "input_text", "highlights": "output_text"})
+    print(df.shape)
+    df.to_csv('C:/Users/gbhat/Documents/GitHub/summarization/data/processed/{}.csv'.format(split))
 
 
 if __name__ == '__name__':