app.py

from flask import Flask, request, jsonify, Response, stream_with_context
from flask_cors import CORS
import os
from dotenv import load_dotenv
import requests
from blog_generator import BlogGenerator
from csv_handler import CSVHandler
import ssl
import logging
from web_scraper import research_topic
import queue
import threading

# Create a logging filter to suppress socket warnings
class SocketWarningFilter(logging.Filter):
    def filter(self, record):
        return not (record.levelname == 'WARNING' and 'socket.send()' in record.getMessage())

# Create a queue for log messages
log_queue = queue.Queue()

# Custom log handler that puts messages in the queue
class QueueHandler(logging.Handler):
    def emit(self, record):
        log_entry = self.format(record)
        log_queue.put(log_entry)

# Set up logging with the custom handler
logger = logging.getLogger()
queue_handler = QueueHandler()
queue_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
queue_handler.addFilter(SocketWarningFilter())  # Add the filter to the handler
logger.addHandler(queue_handler)
logger.setLevel(logging.INFO)

# Also add the filter to the root logger to catch all socket warnings
logging.getLogger().addFilter(SocketWarningFilter())

app = Flask(__name__)
# Enable CORS with specific settings
CORS(app, resources={
    r"/*": {
        "origins": "*",
        "methods": ["GET", "POST", "OPTIONS"],
        "allow_headers": ["Content-Type", "Authorization"]
    }
})

# Configuration
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
PERPLEXITY_API_KEY = os.getenv('PERPLEXITY_API_KEY')
SERPAPI_API_KEY = os.getenv('SERPAPI_API_KEY')

ssl._create_default_https_context = ssl._create_unverified_context

def generate_preliminary_plan(cluster_data):
    logger.info("Generating preliminary plan...")
    try:
        response = requests.post(
            'https://openrouter.ai/api/v1/chat/completions',
            headers={
                'Authorization': f'Bearer {cluster_data["openrouter_key"]}',
                'HTTP-Referer': 'http://127.0.0.1:5001',
                'X-Title': 'Blog Generator'
            },
            json={
                'model': 'google/gemini-2.0-flash-thinking-exp:free',
                'messages': [{
                    'role': 'user',
                    'content': f"""You are part of a team that creates world class blog posts.

For each new blog post project, you are provided with a list of keywords and search intent.

- Keywords: The keywords are to what the blog post is meant to rank for. They are scattered throughout the blog and define the topic of the blog post.

- Search intent: The search intent recognises the intent of the user when searching up the keyword which defines be the theme of the blog post, so they click on our blog to satisfy their search.

- Primary keyword: Out of the keywords, there is one keyword known as the primary keyword. The primary keyword will go in the title and first few sentences. It is important that the topic of the blog post is related to the primary keyword so that you can place it into the title and introduction naturally.

Given a list of keywords and search intent, your job is to understand the goal of the blog post, identify the thought process behind the flow of the blog post and come up with a preliminary plan for the post.

Your output must:
- Recognise the discussion points of the blog post.
- Be in dot point format.

You must ensure that the plan created satisfies the search intent and revolves directly around the given keywords.

When making the plan keep in mind that all keywords must be used in the final blog post.

The final goal of the project is to create a high quality, high value, highly relevant blog post that will satisfy the users search intent and give them everything they need to know about the topic.

A new project just came across your desk with below keywords and search intent:

Keywords:
{cluster_data['Keywords']}

Search intent:
{cluster_data['Intent']}

Primary keyword:
{cluster_data['Primary Keyword']}

Create the preliminary plan."""
                }]
            },
            timeout=60
        )

        logger.info(f"OpenRouter API Response: {response.text}")

        if response.status_code != 200:
            raise Exception(f"OpenRouter API error: {response.text}")

        response_data = response.json()
        if 'choices' not in response_data:
            raise Exception(f"Unexpected API response format: {response_data}")

        return response_data['choices'][0]['message']['content']
    except Exception as e:
        logger.error(f"Error in generate_preliminary_plan: {e}")
        raise

def do_research(plan, openrouter_key):
    logger.info("Doing research...")
    try:
        # Extract key points from plan to create search queries
        plan_lines = [line.strip('* -').strip() for line in plan.split('\n') if line.strip()]

        # Take only the first 3 points
        plan_lines = plan_lines[:3]
        logger.info(f"Researching top 3 points: {plan_lines}")

        all_research = []

        # Research each main point in the plan (limited to 3 points)
        for point in plan_lines:
            if not point:  # Skip empty lines
                continue

            # Get research results including both web content and AI analysis
            # Using 5 sites per point for more comprehensive research
            results = research_topic(point, num_sites=5, openrouter_key=openrouter_key)

            if results['success']:
                all_research.append({
                    'topic': point,
                    'analysis': results['analysis'],
                    'sources': results['sources']
                })

        # Format all research into a comprehensive markdown document
        formatted_research = "# Research Results\n\n"

        for research in all_research:
            formatted_research += f"## {research['topic']}\n\n"
            formatted_research += f"{research['analysis']}\n\n"
            formatted_research += "### Sources Referenced\n\n"

            for source in research['sources']:
                formatted_research += f"- [{source['title']}]({source['source']})\n"
                if source['meta_info']['description']:
                    formatted_research += f"  {source['meta_info']['description']}\n"

            formatted_research += "\n---\n\n"

        return formatted_research

    except Exception as e:
        logger.error(f"Error in do_research: {e}")
        raise

@app.route('/generate-blog', methods=['POST'])
def generate_blog():
    try:
        logger.info("Starting blog generation process for multiple clusters...")

        # Initialize handlers
        blog_gen = BlogGenerator(OPENAI_API_KEY, OPENROUTER_API_KEY)
        csv_handler = CSVHandler()

        generated_blogs = []

        # Get all available clusters
        all_clusters = csv_handler.get_all_clusters()

        for cluster_data in all_clusters:
            try:
                logger.info(f"Processing cluster with primary keyword: {cluster_data['Primary Keyword']}")

                # 2. Generate preliminary plan
                logger.info("Generating preliminary plan...")
                plan = generate_preliminary_plan(cluster_data)

                # 3. Do research
                logger.info("Doing research...")
                research = do_research(plan, OPENROUTER_API_KEY)

                # 4. Create detailed plan
                logger.info("Creating detailed plan...")
                detailed_plan = blog_gen.create_detailed_plan(cluster_data, plan, research)

                # 5. Write blog post
                logger.info("Writing blog post...")
                blog_content = blog_gen.write_blog_post(detailed_plan, cluster_data)

                # 6. Add internal links
                logger.info("Adding internal links...")
                previous_posts = csv_handler.get_previous_posts()
                blog_content = blog_gen.add_internal_links(blog_content, previous_posts)

                # 7. Convert to HTML
                logger.info("Converting to HTML...")
                cover_image_url = blog_gen.get_cover_image(cluster_data['Primary Keyword'])
                html_content = blog_gen.convert_to_html(blog_content, cover_image_url)

                # 8. Generate metadata
                logger.info("Generating metadata...")
                metadata = blog_gen.generate_metadata(blog_content, cluster_data['Primary Keyword'], cluster_data)

                # 9. Get cover image
                logger.info("Getting cover image...")
                cover_image_url = blog_gen.get_cover_image(metadata['title'])

                # Create blog post data
                blog_post_data = {
                    'title': metadata['title'],
                    'slug': metadata['slug'],
                    'meta_description': metadata['meta_description'],
                    'content': html_content,
                    'cover_image': cover_image_url,
                    'keywords': cluster_data['Keywords'],
                    'primary_keyword': cluster_data['Primary Keyword'],
                    'research': research,
                    'detailed_plan': detailed_plan
                }

                # 10. Update tracking CSVs
                logger.info("Updating tracking CSVs...")
                csv_handler.mark_cluster_complete(cluster_data['row_number'])
                csv_handler.log_completed_post({**metadata, 'keywords': cluster_data['Keywords']})

                # Add to generated blogs array
                generated_blogs.append({
                    'status': 'success',
                    'message': f"Blog post generated successfully for {cluster_data['Primary Keyword']}",
                    'data': blog_post_data
                })

            except Exception as e:
                logger.error(f"Error processing cluster {cluster_data['Primary Keyword']}: {e}")
                generated_blogs.append({
                    'status': 'error',
                    'message': f"Failed to generate blog post for {cluster_data['Primary Keyword']}",
                    'error': str(e)
                })

        logger.info("All blog generation completed!")
        return jsonify({
            'status': 'success',
            'message': f'Generated {len(generated_blogs)} blog posts',
            'blogs': generated_blogs
        })

    except Exception as e:
        logger.error(f"Error in generate_blog main process: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/test-api', methods=['GET'])
def test_api():
    try:
        # Test OpenRouter API
        response = requests.post(
            'https://openrouter.ai/api/v1/chat/completions',
            headers={
                'Authorization': f'Bearer {OPENROUTER_API_KEY}',
                'HTTP-Referer': 'http://localhost:5001',
                'X-Title': 'Blog Generator'
            },
            json={
                'model': 'deepseek/deepseek-r1',
                'messages': [{
                    'role': 'user',
                    'content': 'Say hello!'
                }]
            }
        )
        return jsonify({
            'status': 'success',
            'openrouter_response': response.json()
        })
    except Exception as e:
        return jsonify({'error': str(e)}), 500

@app.route('/generate-from-csv', methods=['POST'])
def generate_from_csv():
    try:
        if 'file' not in request.files:
            return jsonify({'error': 'No file uploaded'}), 400

        file = request.files['file']
        if file.filename == '':
            return jsonify({'error': 'No file selected'}), 400

        # Read and decode the CSV content
        csv_content = file.read().decode('utf-8')

        # Initialize handlers
        blog_gen = BlogGenerator(OPENAI_API_KEY, OPENROUTER_API_KEY)
        csv_handler = CSVHandler()

        # Process the uploaded CSV
        clusters = csv_handler.process_uploaded_csv(csv_content)

        if not clusters:
            return jsonify({'error': 'No valid clusters found in CSV'}), 400

        generated_blogs = []

        # Process each cluster
        for cluster_data in clusters:
            try:
                logger.info(f"Processing cluster with primary keyword: {cluster_data['Primary Keyword']}")

                # Generate preliminary plan
                plan = generate_preliminary_plan(cluster_data)

                # Do research
                research = do_research(plan, OPENROUTER_API_KEY)

                # Create detailed plan
                detailed_plan = blog_gen.create_detailed_plan(cluster_data, plan, research)

                # Write blog post
                blog_content = blog_gen.write_blog_post(detailed_plan, cluster_data)

                # Add internal links
                previous_posts = csv_handler.get_previous_posts()
                blog_content = blog_gen.add_internal_links(blog_content, previous_posts)

                # Convert to HTML
                cover_image_url = blog_gen.get_cover_image(cluster_data['Primary Keyword'])
                html_content = blog_gen.convert_to_html(blog_content, cover_image_url)

                # Generate metadata
                metadata = blog_gen.generate_metadata(blog_content, cluster_data['Primary Keyword'], cluster_data)

                # Get cover image
                cover_image_url = blog_gen.get_cover_image(metadata['title'])

                # Create blog post data
                blog_post_data = {
                    'title': metadata['title'],
                    'slug': metadata['slug'],
                    'meta_description': metadata['meta_description'],
                    'content': html_content,
                    'cover_image': cover_image_url,
                    'keywords': cluster_data['Keywords'],
                    'primary_keyword': cluster_data['Primary Keyword'],
                    'research': research,
                    'detailed_plan': detailed_plan
                }

                generated_blogs.append({
                    'status': 'success',
                    'message': f"Blog post generated successfully for {cluster_data['Primary Keyword']}",
                    'data': blog_post_data
                })

            except Exception as e:
                logger.error(f"Error processing cluster {cluster_data['Primary Keyword']}: {e}")
                generated_blogs.append({
                    'status': 'error',
                    'message': f"Failed to generate blog post for {cluster_data['Primary Keyword']}",
                    'error': str(e)
                })

        return jsonify({
            'status': 'success',
            'message': f'Generated {len(generated_blogs)} blog posts from uploaded CSV',
            'blogs': generated_blogs
        })

    except Exception as e:
        logger.error(f"Error in generate_from_csv: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/generate-from-csv-text', methods=['POST'])
def generate_from_csv_text():
    try:
        logger.info("Starting blog generation process for multiple clusters...")

        # Get CSV content and OpenRouter API key from request JSON
        data = request.get_json()
        if not data or 'csv_content' not in data:
            return jsonify({'error': 'No CSV content provided'}), 400
        if 'openrouter_key' not in data:
            return jsonify({'error': 'OpenRouter API key is required'}), 400

        csv_content = data['csv_content']
        openrouter_key = data['openrouter_key']

        # Initialize handlers with the provided OpenRouter key
        blog_gen = BlogGenerator(OPENAI_API_KEY, openrouter_key)
        csv_handler = CSVHandler()

        # Process the CSV text
        clusters = csv_handler.process_csv_text(csv_content)

        if not clusters:
            return jsonify({'error': 'No valid clusters found in CSV'}), 400

        generated_blogs = []

        # Process each cluster
        for cluster_data in clusters:
            try:
                logger.info(f"Processing cluster with primary keyword: {cluster_data['Primary Keyword']}")

                # Add OpenRouter key to cluster_data for use in functions
                cluster_data['openrouter_key'] = openrouter_key

                # Generate preliminary plan
                logger.info("Generating preliminary plan...")
                plan = generate_preliminary_plan(cluster_data)

                # Do research
                logger.info("Doing research...")
                research = do_research(plan, openrouter_key)

                # Create detailed plan
                logger.info("Creating detailed plan...")
                detailed_plan = blog_gen.create_detailed_plan(cluster_data, plan, research)

                # Write blog post
                logger.info("Writing blog post...")
                blog_content = blog_gen.write_blog_post(detailed_plan, cluster_data)

                # Add internal links
                logger.info("Adding internal links...")
                previous_posts = csv_handler.get_previous_posts()
                blog_content = blog_gen.add_internal_links(blog_content, previous_posts)

                # Convert to HTML
                logger.info("Converting to HTML...")
                cover_image_url = blog_gen.get_cover_image(cluster_data['Primary Keyword'])
                html_content = blog_gen.convert_to_html(blog_content, cover_image_url)

                # Generate metadata
                logger.info("Generating metadata...")
                metadata = blog_gen.generate_metadata(blog_content, cluster_data['Primary Keyword'], cluster_data)

                # Get cover image
                logger.info("Getting cover image...")
                cover_image_url = blog_gen.get_cover_image(metadata['title'])

                blog_post_data = {
                    'title': metadata['title'],
                    'slug': metadata['slug'],
                    'meta_description': metadata['meta_description'],
                    'content': html_content,
                    'cover_image': cover_image_url,
                    'keywords': cluster_data['Keywords'],
                    'primary_keyword': cluster_data['Primary Keyword'],
                    'research': research,
                    'detailed_plan': detailed_plan
                }

                generated_blogs.append({
                    'status': 'success',
                    'message': f"Blog post generated successfully for {cluster_data['Primary Keyword']}",
                    'data': blog_post_data
                })

                csv_handler.mark_cluster_complete(cluster_data['row_number'])
                csv_handler.log_completed_post({**metadata, 'keywords': cluster_data['Keywords']})

            except Exception as e:
                logger.error(f"Error processing cluster {cluster_data['Primary Keyword']}: {e}")
                generated_blogs.append({
                    'status': 'error',
                    'message': f"Failed to generate blog post for {cluster_data['Primary Keyword']}",
                    'error': str(e)
                })

        logger.info("All blog generation completed!")
        return jsonify({
            'status': 'success',
            'message': f'Generated {len(generated_blogs)} blog posts from CSV text',
            'blogs': generated_blogs
        })

    except Exception as e:
        logger.error(f"Error in generate_from_csv_text: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/logs/stream')
def stream_logs():
    def generate():
        while True:
            try:
                # Get log message from queue, no timeout here, handling is done in the generator
                log_message = log_queue.get(timeout=1)  # Adding timeout of 1 second to avoid blocking indefinitely
                yield f"data: {log_message}\n\n"
            except queue.Empty:
                # Send a heartbeat to keep the connection alive
                yield "data: heartbeat\n\n"
            except GeneratorExit:
                break

    response = Response(stream_with_context(generate()), mimetype='text/event-stream')
    response.headers['Cache-Control'] = 'no-cache'
    response.headers['Connection'] = 'keep-alive'
    response.headers['Access-Control-Allow-Origin'] = '*'
    return response