Pamudu13 commited on
Commit
49c4b1b
·
verified ·
1 Parent(s): b556016

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +97 -234
app.py CHANGED
@@ -10,11 +10,6 @@ import logging
10
  from web_scraper import research_topic
11
  import queue
12
  import threading
13
- import aiohttp
14
- import asyncio
15
- from quart import Quart, request, jsonify
16
- from flask_socketio import SocketIO, emit
17
- import eventlet
18
 
19
  # Create a queue for log messages
20
  log_queue = queue.Queue()
@@ -32,23 +27,16 @@ queue_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(me
32
  logger.addHandler(queue_handler)
33
  logger.setLevel(logging.INFO)
34
 
35
- # Initialize Flask app with eventlet
36
- eventlet.monkey_patch()
37
  app = Flask(__name__)
38
-
39
- # Configure CORS with WebSocket support
40
  CORS(app, resources={
41
  r"/*": {
42
- "origins": ["http://localhost:8000", "http://127.0.0.1:8000"], # Add your Laravel domains
43
  "methods": ["GET", "POST", "OPTIONS"],
44
- "allow_headers": ["Content-Type", "Authorization"],
45
- "supports_credentials": True
46
  }
47
  })
48
 
49
- # Initialize SocketIO with CORS support
50
- socketio = SocketIO(app, cors_allowed_origins=["http://localhost:8000", "http://127.0.0.1:8000"], async_mode='eventlet')
51
-
52
  # Configuration
53
  OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
54
  OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
@@ -57,39 +45,6 @@ SERPAPI_API_KEY = os.getenv('SERPAPI_API_KEY')
57
 
58
  ssl._create_default_https_context = ssl._create_unverified_context
59
 
60
- # WebSocket event handlers
61
- @socketio.on('connect')
62
- def handle_connect():
63
- logger.info("Client connected")
64
- emit('connection_response', {'status': 'connected'})
65
-
66
- @socketio.on('disconnect')
67
- def handle_disconnect():
68
- logger.info("Client disconnected")
69
-
70
- @socketio.on_error()
71
- def error_handler(e):
72
- logger.error(f"SocketIO error: {str(e)}")
73
- emit('error', {'error': str(e)})
74
-
75
- # Modified stream_logs route to use WebSocket
76
- @socketio.on('subscribe_logs')
77
- def handle_subscribe_logs():
78
- try:
79
- while True:
80
- try:
81
- log_message = log_queue.get(timeout=1)
82
- emit('log_message', {'message': log_message})
83
- except queue.Empty:
84
- emit('heartbeat', {'message': 'heartbeat'})
85
- except Exception as e:
86
- logger.error(f"Error in log streaming: {e}")
87
- emit('error', {'error': str(e)})
88
- eventlet.sleep(0.1) # Prevent CPU overuse
89
- except Exception as e:
90
- logger.error(f"Error in log subscription: {e}")
91
- emit('error', {'error': str(e)})
92
-
93
  def generate_preliminary_plan(cluster_data):
94
  logger.info("Generating preliminary plan...")
95
  try:
@@ -421,125 +376,14 @@ def generate_from_csv():
421
  logger.error(f"Error in generate_from_csv: {e}")
422
  return jsonify({'error': str(e)}), 500
423
 
424
- async def generate_preliminary_plan_async(cluster_data):
425
- logger.info("Generating preliminary plan...")
426
- try:
427
- async with aiohttp.ClientSession() as session:
428
- async with session.post(
429
- 'https://openrouter.ai/api/v1/chat/completions',
430
- headers={
431
- 'Authorization': f'Bearer {cluster_data["openrouter_key"]}',
432
- 'HTTP-Referer': 'http://127.0.0.1:5001',
433
- 'X-Title': 'Blog Generator'
434
- },
435
- json={
436
- 'model': 'google/gemini-2.0-flash-thinking-exp:free',
437
- 'messages': [{
438
- 'role': 'user',
439
- 'content': f"""You are part of a team that creates world class blog posts.
440
-
441
- For each new blog post project, you are provided with a list of keywords and search intent.
442
-
443
- - Keywords: The keywords are to what the blog post is meant to rank for. They are scattered throughout the blog and define the topic of the blog post.
444
-
445
- - Search intent: The search intent recognises the intent of the user when searching up the keyword which defines be the theme of the blog post, so they click on our blog to satisfy their search.
446
-
447
- - Primary keyword: Out of the keywords, there is one keyword known as the primary keyword. The primary keyword will go in the title and first few sentences. It is important that the topic of the blog post is related to the primary keyword so that you can place it into the title and introduction naturally.
448
-
449
- Given a list of keywords and search intent, your job is to understand the goal of the blog post, identify the thought process behind the flow of the blog post and come up with a preliminary plan for the post.
450
-
451
- Your output must:
452
- - Recognise the discussion points of the blog post.
453
- - Be in dot point format.
454
-
455
- You must ensure that the plan created satisfies the search intent and revolves directly around the given keywords.
456
-
457
- When making the plan keep in mind that all keywords must be used in the final blog post.
458
-
459
- The final goal of the project is to create a high quality, high value, highly relevant blog post that will satisfy the users search intent and give them everything they need to know about the topic.
460
-
461
- A new project just came across your desk with below keywords and search intent:
462
-
463
- Keywords:
464
- {cluster_data['Keywords']}
465
-
466
- Search intent:
467
- {cluster_data['Intent']}
468
-
469
- Primary keyword:
470
- {cluster_data['Primary Keyword']}
471
-
472
- Create the preliminary plan."""
473
- }]
474
- }
475
- ) as response:
476
- if response.status != 200:
477
- raise Exception(f"OpenRouter API error: {await response.text()}")
478
-
479
- response_data = await response.json()
480
- if 'choices' not in response_data:
481
- raise Exception(f"Unexpected API response format: {response_data}")
482
-
483
- return response_data['choices'][0]['message']['content']
484
- except Exception as e:
485
- logger.error(f"Error in generate_preliminary_plan_async: {e}")
486
- raise
487
-
488
- async def do_research_async(plan, openrouter_key):
489
- logger.info("Doing research...")
490
- try:
491
- # Extract key points from plan to create search queries
492
- plan_lines = [line.strip('* -').strip() for line in plan.split('\n') if line.strip()]
493
- plan_lines = plan_lines[:3]
494
- logger.info(f"Researching top 3 points: {plan_lines}")
495
-
496
- all_research = []
497
- async with aiohttp.ClientSession() as session:
498
- research_tasks = []
499
- for point in plan_lines:
500
- if point:
501
- research_tasks.append(
502
- asyncio.create_task(
503
- research_topic_async(point, num_sites=5, openrouter_key=openrouter_key, session=session)
504
- )
505
- )
506
-
507
- results = await asyncio.gather(*research_tasks)
508
-
509
- for result in results:
510
- if result['success']:
511
- all_research.append({
512
- 'topic': result['query'],
513
- 'analysis': result['analysis'],
514
- 'sources': result['sources']
515
- })
516
-
517
- formatted_research = "# Research Results\n\n"
518
- for research in all_research:
519
- formatted_research += f"## {research['topic']}\n\n"
520
- formatted_research += f"{research['analysis']}\n\n"
521
- formatted_research += "### Sources Referenced\n\n"
522
-
523
- for source in research['sources']:
524
- formatted_research += f"- [{source['title']}]({source['source']})\n"
525
- if source['meta_info']['description']:
526
- formatted_research += f" {source['meta_info']['description']}\n"
527
-
528
- formatted_research += "\n---\n\n"
529
-
530
- return formatted_research
531
-
532
- except Exception as e:
533
- logger.error(f"Error in do_research_async: {e}")
534
- raise
535
 
536
  @app.route('/generate-from-csv-text', methods=['POST'])
537
- async def generate_from_csv_text():
538
  try:
539
  logger.info("Starting blog generation process for multiple clusters...")
540
 
541
  # Get CSV content and OpenRouter API key from request JSON
542
- data = await request.get_json()
543
  if not data or 'csv_content' not in data:
544
  return jsonify({'error': 'No CSV content provided'}), 400
545
  if 'openrouter_key' not in data:
@@ -560,77 +404,76 @@ async def generate_from_csv_text():
560
 
561
  generated_blogs = []
562
 
563
- # Process each cluster asynchronously
564
- async with aiohttp.ClientSession() as session:
565
- for cluster_data in clusters:
566
- try:
567
- logger.info(f"Processing cluster with primary keyword: {cluster_data['Primary Keyword']}")
568
-
569
- # Add OpenRouter key to cluster_data for use in functions
570
- cluster_data['openrouter_key'] = openrouter_key
571
-
572
- # Generate preliminary plan
573
- logger.info("Generating preliminary plan...")
574
- plan = await generate_preliminary_plan_async(cluster_data)
575
-
576
- # Do research
577
- logger.info("Doing research...")
578
- research = await do_research_async(plan, openrouter_key)
579
-
580
- # Create detailed plan
581
- logger.info("Creating detailed plan...")
582
- detailed_plan = await blog_gen.create_detailed_plan_async(cluster_data, plan, research)
583
-
584
- # Write blog post
585
- logger.info("Writing blog post...")
586
- blog_content = await blog_gen.write_blog_post_async(detailed_plan, cluster_data)
587
-
588
- # Add internal links
589
- logger.info("Adding internal links...")
590
- previous_posts = csv_handler.get_previous_posts()
591
- blog_content = await blog_gen.add_internal_links_async(blog_content, previous_posts)
592
-
593
- # Convert to HTML
594
- logger.info("Converting to HTML...")
595
- cover_image_url = await blog_gen.get_cover_image_async(cluster_data['Primary Keyword'])
596
- html_content = await blog_gen.convert_to_html_async(blog_content, cover_image_url)
597
-
598
- # Generate metadata
599
- logger.info("Generating metadata...")
600
- metadata = await blog_gen.generate_metadata_async(blog_content, cluster_data['Primary Keyword'], cluster_data)
601
-
602
- # Get cover image
603
- logger.info("Getting cover image...")
604
- cover_image_url = await blog_gen.get_cover_image_async(metadata['title'])
605
-
606
- blog_post_data = {
607
- 'title': metadata['title'],
608
- 'slug': metadata['slug'],
609
- 'meta_description': metadata['meta_description'],
610
- 'content': html_content,
611
- 'cover_image': cover_image_url,
612
- 'keywords': cluster_data['Keywords'],
613
- 'primary_keyword': cluster_data['Primary Keyword'],
614
- 'research': research,
615
- 'detailed_plan': detailed_plan
616
- }
617
-
618
- generated_blogs.append({
619
- 'status': 'success',
620
- 'message': f"Blog post generated successfully for {cluster_data['Primary Keyword']}",
621
- 'data': blog_post_data
622
- })
623
-
624
- csv_handler.mark_cluster_complete(cluster_data['row_number'])
625
- csv_handler.log_completed_post({**metadata, 'keywords': cluster_data['Keywords']})
626
-
627
- except Exception as e:
628
- logger.error(f"Error processing cluster {cluster_data['Primary Keyword']}: {e}")
629
- generated_blogs.append({
630
- 'status': 'error',
631
- 'message': f"Failed to generate blog post for {cluster_data['Primary Keyword']}",
632
- 'error': str(e)
633
- })
634
 
635
  logger.info("All blog generation completed!")
636
  return jsonify({
@@ -643,6 +486,26 @@ async def generate_from_csv_text():
643
  logger.error(f"Error in generate_from_csv_text: {e}")
644
  return jsonify({'error': str(e)}), 500
645
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
646
  if __name__ == '__main__':
647
- logger.info("Starting Flask API server with SocketIO...")
648
- socketio.run(app, host='127.0.0.1', port=5001, debug=True)
 
10
  from web_scraper import research_topic
11
  import queue
12
  import threading
 
 
 
 
 
13
 
14
  # Create a queue for log messages
15
  log_queue = queue.Queue()
 
27
  logger.addHandler(queue_handler)
28
  logger.setLevel(logging.INFO)
29
 
 
 
30
  app = Flask(__name__)
31
+ # Enable CORS with specific settings
 
32
  CORS(app, resources={
33
  r"/*": {
34
+ "origins": "*",
35
  "methods": ["GET", "POST", "OPTIONS"],
36
+ "allow_headers": ["Content-Type", "Authorization"]
 
37
  }
38
  })
39
 
 
 
 
40
  # Configuration
41
  OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
42
  OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
 
45
 
46
  ssl._create_default_https_context = ssl._create_unverified_context
47
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  def generate_preliminary_plan(cluster_data):
49
  logger.info("Generating preliminary plan...")
50
  try:
 
376
  logger.error(f"Error in generate_from_csv: {e}")
377
  return jsonify({'error': str(e)}), 500
378
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
379
 
380
  @app.route('/generate-from-csv-text', methods=['POST'])
381
+ def generate_from_csv_text():
382
  try:
383
  logger.info("Starting blog generation process for multiple clusters...")
384
 
385
  # Get CSV content and OpenRouter API key from request JSON
386
+ data = request.get_json()
387
  if not data or 'csv_content' not in data:
388
  return jsonify({'error': 'No CSV content provided'}), 400
389
  if 'openrouter_key' not in data:
 
404
 
405
  generated_blogs = []
406
 
407
+ # Process each cluster
408
+ for cluster_data in clusters:
409
+ try:
410
+ logger.info(f"Processing cluster with primary keyword: {cluster_data['Primary Keyword']}")
411
+
412
+ # Add OpenRouter key to cluster_data for use in functions
413
+ cluster_data['openrouter_key'] = openrouter_key
414
+
415
+ # Generate preliminary plan
416
+ logger.info("Generating preliminary plan...")
417
+ plan = generate_preliminary_plan(cluster_data)
418
+
419
+ # Do research
420
+ logger.info("Doing research...")
421
+ research = do_research(plan, openrouter_key)
422
+
423
+ # Create detailed plan
424
+ logger.info("Creating detailed plan...")
425
+ detailed_plan = blog_gen.create_detailed_plan(cluster_data, plan, research)
426
+
427
+ # Write blog post
428
+ logger.info("Writing blog post...")
429
+ blog_content = blog_gen.write_blog_post(detailed_plan, cluster_data)
430
+
431
+ # Add internal links
432
+ logger.info("Adding internal links...")
433
+ previous_posts = csv_handler.get_previous_posts()
434
+ blog_content = blog_gen.add_internal_links(blog_content, previous_posts)
435
+
436
+ # Convert to HTML
437
+ logger.info("Converting to HTML...")
438
+ cover_image_url = blog_gen.get_cover_image(cluster_data['Primary Keyword'])
439
+ html_content = blog_gen.convert_to_html(blog_content, cover_image_url)
440
+
441
+ # Generate metadata
442
+ logger.info("Generating metadata...")
443
+ metadata = blog_gen.generate_metadata(blog_content, cluster_data['Primary Keyword'], cluster_data)
444
+
445
+ # Get cover image
446
+ logger.info("Getting cover image...")
447
+ cover_image_url = blog_gen.get_cover_image(metadata['title'])
448
+
449
+ blog_post_data = {
450
+ 'title': metadata['title'],
451
+ 'slug': metadata['slug'],
452
+ 'meta_description': metadata['meta_description'],
453
+ 'content': html_content,
454
+ 'cover_image': cover_image_url,
455
+ 'keywords': cluster_data['Keywords'],
456
+ 'primary_keyword': cluster_data['Primary Keyword'],
457
+ 'research': research,
458
+ 'detailed_plan': detailed_plan
459
+ }
460
+
461
+ generated_blogs.append({
462
+ 'status': 'success',
463
+ 'message': f"Blog post generated successfully for {cluster_data['Primary Keyword']}",
464
+ 'data': blog_post_data
465
+ })
466
+
467
+ csv_handler.mark_cluster_complete(cluster_data['row_number'])
468
+ csv_handler.log_completed_post({**metadata, 'keywords': cluster_data['Keywords']})
469
+
470
+ except Exception as e:
471
+ logger.error(f"Error processing cluster {cluster_data['Primary Keyword']}: {e}")
472
+ generated_blogs.append({
473
+ 'status': 'error',
474
+ 'message': f"Failed to generate blog post for {cluster_data['Primary Keyword']}",
475
+ 'error': str(e)
476
+ })
 
477
 
478
  logger.info("All blog generation completed!")
479
  return jsonify({
 
486
  logger.error(f"Error in generate_from_csv_text: {e}")
487
  return jsonify({'error': str(e)}), 500
488
 
489
+ @app.route('/logs/stream')
490
+ def stream_logs():
491
+ def generate():
492
+ while True:
493
+ try:
494
+ # Get log message from queue, timeout after 1 second
495
+ log_message = log_queue.get(timeout=1)
496
+ yield f"data: {log_message}\n\n"
497
+ except queue.Empty:
498
+ # Send a heartbeat to keep the connection alive
499
+ yield "data: heartbeat\n\n"
500
+ except GeneratorExit:
501
+ break
502
+
503
+ response = Response(stream_with_context(generate()), mimetype='text/event-stream')
504
+ response.headers['Cache-Control'] = 'no-cache'
505
+ response.headers['Connection'] = 'keep-alive'
506
+ response.headers['Access-Control-Allow-Origin'] = '*'
507
+ return response
508
+
509
  if __name__ == '__main__':
510
+ logger.info("Starting Flask API server...")
511
+ app.run(host='127.0.0.1', port=5001, debug=True, threaded=True)