utils/scrap_greed_fear_index.py

import requests
import pandas as pd
import os
from dotenv import load_dotenv
import logging

# Load environment variables
load_dotenv()
api_key = os.getenv("RAPIDAPI_KEY")

# Setup logging
log_folder = os.getenv("LOG_FOLDER")
os.makedirs(log_folder, exist_ok=True)
log_file = os.path.join(log_folder, "greed_fear_index.log")
logging.basicConfig(filename=log_file, level=logging.INFO, format="%(asctime)s [%(levelname)s] - %(message)s")

def fetch_greed_fear_index():
    url = "https://fear-and-greed-index.p.rapidapi.com/v1/fgi"
    headers = {
        "X-RapidAPI-Key": api_key,
        "X-RapidAPI-Host": "fear-and-greed-index.p.rapidapi.com"
    }

    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        data = response.json()
        df = pd.json_normalize(data, sep='_')
        df.to_csv('output/greed_fear_index.csv', index=False)
        logging.info("Greed and Fear index data fetched successfully.")
    else:
        logging.error(f"Failed to fetch data: {response.text}")

if __name__ == "__main__":
    fetch_greed_fear_index()