Update

dvc.yaml

@@ -3,6 +3,7 @@ stages:
     cmd: python src/models/train_model.py
     deps:
       - data/processed/train.csv
+      - data/processed/validation.csv
       - src/models/train_model.py
     outs:
       - models:

src/data/make_dataset.py

@@ -6,9 +6,9 @@ def make_dataset(dataset='cnn_dailymail', split='train'):
     """make dataset for summarisation"""
     dataset = load_dataset(dataset, '3.0.0', split=split)
     df = pd.DataFrame()
-    df['input_text'] = dataset['article']
-    df['output_text'] = dataset['highlights']
-    df.to_csv('C:/Users/gbhat/Documents/GitHub/summarization/data/processed/{}.csv'.format(split, split))
+    df['article'] = dataset['article']
+    df['highlights'] = dataset['highlights']
+    df.to_csv('summarization/data/raw/{}.csv'.format(split))
 
 
 if __name__ == '__main__':

src/data/process_data.py

@@ -0,0 +1,8 @@
+import pandas as pd
+
+def process_data(split='train'):
+    df= pd.DataFrame()
+    dataset = pd.load_csv('summarization/data/raw/{}.csv'.format(split))
+    df['article'] = dataset['article']
+    df['highlights'] = dataset['highlights']
+    df.to_csv('summarization/data/processed/{}.csv'.format(split))