Update app.py

app.py

@@ -10,6 +10,9 @@ import logging
 from web_scraper import research_topic
 import queue
 import threading
+import aiohttp
+import asyncio
+from quart import Quart, request, jsonify
 
 # Create a queue for log messages
 log_queue = queue.Queue()
@@ -376,14 +379,125 @@ def generate_from_csv():
         logger.error(f"Error in generate_from_csv: {e}")
         return jsonify({'error': str(e)}), 500
 
+async def generate_preliminary_plan_async(cluster_data):
+    logger.info("Generating preliminary plan...")
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                'https://openrouter.ai/api/v1/chat/completions',
+                headers={
+                    'Authorization': f'Bearer {cluster_data["openrouter_key"]}',
+                    'HTTP-Referer': 'http://127.0.0.1:5001',
+                    'X-Title': 'Blog Generator'
+                },
+                json={
+                    'model': 'google/gemini-2.0-flash-thinking-exp:free',
+                    'messages': [{
+                        'role': 'user',
+                        'content': f"""You are part of a team that creates world class blog posts.
+
+For each new blog post project, you are provided with a list of keywords and search intent.
+
+- Keywords: The keywords are to what the blog post is meant to rank for. They are scattered throughout the blog and define the topic of the blog post.
+
+- Search intent: The search intent recognises the intent of the user when searching up the keyword which defines be the theme of the blog post, so they click on our blog to satisfy their search.
+
+- Primary keyword: Out of the keywords, there is one keyword known as the primary keyword. The primary keyword will go in the title and first few sentences. It is important that the topic of the blog post is related to the primary keyword so that you can place it into the title and introduction naturally.
+
+Given a list of keywords and search intent, your job is to understand the goal of the blog post, identify the thought process behind the flow of the blog post and come up with a preliminary plan for the post.
+
+Your output must:
+- Recognise the discussion points of the blog post.
+- Be in dot point format.
+
+You must ensure that the plan created satisfies the search intent and revolves directly around the given keywords.
+
+When making the plan keep in mind that all keywords must be used in the final blog post.
+
+The final goal of the project is to create a high quality, high value, highly relevant blog post that will satisfy the users search intent and give them everything they need to know about the topic.
+
+A new project just came across your desk with below keywords and search intent:
+
+Keywords:
+{cluster_data['Keywords']}
+
+Search intent:
+{cluster_data['Intent']}
+
+Primary keyword:
+{cluster_data['Primary Keyword']}
+
+Create the preliminary plan."""
+                    }]
+                }
+            ) as response:
+                if response.status != 200:
+                    raise Exception(f"OpenRouter API error: {await response.text()}")
+
+                response_data = await response.json()
+                if 'choices' not in response_data:
+                    raise Exception(f"Unexpected API response format: {response_data}")
+
+                return response_data['choices'][0]['message']['content']
+    except Exception as e:
+        logger.error(f"Error in generate_preliminary_plan_async: {e}")
+        raise
+
+async def do_research_async(plan, openrouter_key):
+    logger.info("Doing research...")
+    try:
+        # Extract key points from plan to create search queries
+        plan_lines = [line.strip('* -').strip() for line in plan.split('\n') if line.strip()]
+        plan_lines = plan_lines[:3]
+        logger.info(f"Researching top 3 points: {plan_lines}")
+
+        all_research = []
+        async with aiohttp.ClientSession() as session:
+            research_tasks = []
+            for point in plan_lines:
+                if point:
+                    research_tasks.append(
+                        asyncio.create_task(
+                            research_topic_async(point, num_sites=5, openrouter_key=openrouter_key, session=session)
+                        )
+                    )
+
+            results = await asyncio.gather(*research_tasks)
+
+            for result in results:
+                if result['success']:
+                    all_research.append({
+                        'topic': result['query'],
+                        'analysis': result['analysis'],
+                        'sources': result['sources']
+                    })
+
+        formatted_research = "# Research Results\n\n"
+        for research in all_research:
+            formatted_research += f"## {research['topic']}\n\n"
+            formatted_research += f"{research['analysis']}\n\n"
+            formatted_research += "### Sources Referenced\n\n"
+
+            for source in research['sources']:
+                formatted_research += f"- [{source['title']}]({source['source']})\n"
+                if source['meta_info']['description']:
+                    formatted_research += f"  {source['meta_info']['description']}\n"
+
+            formatted_research += "\n---\n\n"
+
+        return formatted_research
+
+    except Exception as e:
+        logger.error(f"Error in do_research_async: {e}")
+        raise
 
 @app.route('/generate-from-csv-text', methods=['POST'])
-def generate_from_csv_text():
+async def generate_from_csv_text():
     try:
         logger.info("Starting blog generation process for multiple clusters...")
 
         # Get CSV content and OpenRouter API key from request JSON
-        data = request.get_json()
+        data = await request.get_json()
         if not data or 'csv_content' not in data:
             return jsonify({'error': 'No CSV content provided'}), 400
         if 'openrouter_key' not in data:
@@ -404,76 +518,77 @@ def generate_from_csv_text():
 
         generated_blogs = []
 
-        # Process each cluster
-        for cluster_data in clusters:
-            try:
-                logger.info(f"Processing cluster with primary keyword: {cluster_data['Primary Keyword']}")
-
-                # Add OpenRouter key to cluster_data for use in functions
-                cluster_data['openrouter_key'] = openrouter_key
-
-                # Generate preliminary plan
-                logger.info("Generating preliminary plan...")
-                plan = generate_preliminary_plan(cluster_data)
-
-                # Do research
-                logger.info("Doing research...")
-                research = do_research(plan, openrouter_key)
-
-                # Create detailed plan
-                logger.info("Creating detailed plan...")
-                detailed_plan = blog_gen.create_detailed_plan(cluster_data, plan, research)
-
-                # Write blog post
-                logger.info("Writing blog post...")
-                blog_content = blog_gen.write_blog_post(detailed_plan, cluster_data)
-
-                # Add internal links
-                logger.info("Adding internal links...")
-                previous_posts = csv_handler.get_previous_posts()
-                blog_content = blog_gen.add_internal_links(blog_content, previous_posts)
-
-                # Convert to HTML
-                logger.info("Converting to HTML...")
-                cover_image_url = blog_gen.get_cover_image(cluster_data['Primary Keyword'])
-                html_content = blog_gen.convert_to_html(blog_content, cover_image_url)
-
-                # Generate metadata
-                logger.info("Generating metadata...")
-                metadata = blog_gen.generate_metadata(blog_content, cluster_data['Primary Keyword'], cluster_data)
-
-                # Get cover image
-                logger.info("Getting cover image...")
-                cover_image_url = blog_gen.get_cover_image(metadata['title'])
-
-                blog_post_data = {
-                    'title': metadata['title'],
-                    'slug': metadata['slug'],
-                    'meta_description': metadata['meta_description'],
-                    'content': html_content,
-                    'cover_image': cover_image_url,
-                    'keywords': cluster_data['Keywords'],
-                    'primary_keyword': cluster_data['Primary Keyword'],
-                    'research': research,
-                    'detailed_plan': detailed_plan
-                }
-
-                generated_blogs.append({
-                    'status': 'success',
-                    'message': f"Blog post generated successfully for {cluster_data['Primary Keyword']}",
-                    'data': blog_post_data
-                })
-
-                csv_handler.mark_cluster_complete(cluster_data['row_number'])
-                csv_handler.log_completed_post({**metadata, 'keywords': cluster_data['Keywords']})
-
-            except Exception as e:
-                logger.error(f"Error processing cluster {cluster_data['Primary Keyword']}: {e}")
-                generated_blogs.append({
-                    'status': 'error',
-                    'message': f"Failed to generate blog post for {cluster_data['Primary Keyword']}",
-                    'error': str(e)
-                })
+        # Process each cluster asynchronously
+        async with aiohttp.ClientSession() as session:
+            for cluster_data in clusters:
+                try:
+                    logger.info(f"Processing cluster with primary keyword: {cluster_data['Primary Keyword']}")
+
+                    # Add OpenRouter key to cluster_data for use in functions
+                    cluster_data['openrouter_key'] = openrouter_key
+
+                    # Generate preliminary plan
+                    logger.info("Generating preliminary plan...")
+                    plan = await generate_preliminary_plan_async(cluster_data)
+
+                    # Do research
+                    logger.info("Doing research...")
+                    research = await do_research_async(plan, openrouter_key)
+
+                    # Create detailed plan
+                    logger.info("Creating detailed plan...")
+                    detailed_plan = await blog_gen.create_detailed_plan_async(cluster_data, plan, research)
+
+                    # Write blog post
+                    logger.info("Writing blog post...")
+                    blog_content = await blog_gen.write_blog_post_async(detailed_plan, cluster_data)
+
+                    # Add internal links
+                    logger.info("Adding internal links...")
+                    previous_posts = csv_handler.get_previous_posts()
+                    blog_content = await blog_gen.add_internal_links_async(blog_content, previous_posts)
+
+                    # Convert to HTML
+                    logger.info("Converting to HTML...")
+                    cover_image_url = await blog_gen.get_cover_image_async(cluster_data['Primary Keyword'])
+                    html_content = await blog_gen.convert_to_html_async(blog_content, cover_image_url)
+
+                    # Generate metadata
+                    logger.info("Generating metadata...")
+                    metadata = await blog_gen.generate_metadata_async(blog_content, cluster_data['Primary Keyword'], cluster_data)
+
+                    # Get cover image
+                    logger.info("Getting cover image...")
+                    cover_image_url = await blog_gen.get_cover_image_async(metadata['title'])
+
+                    blog_post_data = {
+                        'title': metadata['title'],
+                        'slug': metadata['slug'],
+                        'meta_description': metadata['meta_description'],
+                        'content': html_content,
+                        'cover_image': cover_image_url,
+                        'keywords': cluster_data['Keywords'],
+                        'primary_keyword': cluster_data['Primary Keyword'],
+                        'research': research,
+                        'detailed_plan': detailed_plan
+                    }
+
+                    generated_blogs.append({
+                        'status': 'success',
+                        'message': f"Blog post generated successfully for {cluster_data['Primary Keyword']}",
+                        'data': blog_post_data
+                    })
+
+                    csv_handler.mark_cluster_complete(cluster_data['row_number'])
+                    csv_handler.log_completed_post({**metadata, 'keywords': cluster_data['Keywords']})
+
+                except Exception as e:
+                    logger.error(f"Error processing cluster {cluster_data['Primary Keyword']}: {e}")
+                    generated_blogs.append({
+                        'status': 'error',
+                        'message': f"Failed to generate blog post for {cluster_data['Primary Keyword']}",
+                        'error': str(e)
+                    })
 
         logger.info("All blog generation completed!")
         return jsonify({
@@ -492,7 +607,7 @@ def stream_logs():
         while True:
             try:
                 # Get log message from queue, timeout after 1 second
-                log_message = log_queue.get(timeout=10)
+                log_message = log_queue.get(timeout=1)
                 yield f"data: {log_message}\n\n"
             except queue.Empty:
                 # Send a heartbeat to keep the connection alive
@@ -508,4 +623,10 @@ def stream_logs():
 
 if __name__ == '__main__':
     logger.info("Starting Flask API server...")
-    app.run(host='127.0.0.1', port=5001, debug=True)
+    import hypercorn.asyncio
+    import hypercorn.config
+
+    config = hypercorn.config.Config()
+    config.bind = ["127.0.0.1:5001"]
+
+    asyncio.run(hypercorn.asyncio.serve(app, config))