Spaces:
Running
Running
File size: 5,580 Bytes
bdafe83 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 |
import glob
import logging
import os
import sys
from argparse import Namespace
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from arguments import parse_args
from agentreview.experiment_config import all_settings
from agentreview.agent import Player
from agentreview.environments import PaperReview
from agentreview.paper_review_settings import get_experiment_settings
from agentreview.paper_review_arena import PaperReviewArena
from agentreview.paper_review_player import PaperExtractorPlayer, AreaChair, Reviewer
from agentreview.role_descriptions import get_ac_config, get_reviewer_player_config, get_author_config, \
get_paper_extractor_config
from utility.utils import project_setup, get_paper_decision_mapping
# Set up logging configuration
logging.basicConfig(
level=logging.DEBUG, # Set to DEBUG to capture all levels of logs
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout) # Output to stdout
]
)
logger = logging.getLogger(__name__)
# Sample Paper IDs from each category
def main(args: Namespace):
"""
Main routine for paper review and rebuttals:
* Phase 1: Reviewer Assessment (Reviewer writes reviews)
* Phase 2: Author-Reviewer Discussion. (Author writes rebuttals).
* Phase 3: Reviewer-AC Discussion.
* Phase 4: Meta-Review Compilation. (AC writes metareviews)
Args:
args (Namespace): Parsed arguments for configuring the review process.
"""
args.task = "paper_review"
paper_id2decision, paper_decision2ids = get_paper_decision_mapping(args.data_dir, args.conference)
# Sample paper IDs for the simulation from existing data.
paper_paths = glob.glob(os.path.join(args.data_dir, args.conference, "paper", "**", "*.pdf"))
sampled_paper_ids = [int(os.path.basename(p).split(".pdf")[0]) for p in paper_paths if p.endswith(".pdf")]
for paper_id in sampled_paper_ids:
experiment_setting = get_experiment_settings(all_settings[args.experiment_name])
# Ground-truth decision in the conference.
# We use this to partition the papers into different quality.
paper_decision = paper_id2decision[paper_id]
logger.info(f"Experiment Started")
logger.info(f"Paper ID: {paper_id} ({paper_decision})")
player_names, players = [], []
for role, players_list in experiment_setting["players"].items():
for i, player_config in enumerate(players_list):
if role == "Paper Extractor":
player_config = get_paper_extractor_config(global_settings=experiment_setting['global_settings'], )
player = PaperExtractorPlayer(data_dir=args.data_dir, paper_id=paper_id,
paper_decision=paper_decision,
args=args,
conference=args.conference, **player_config)
player_names.append(player.name)
elif role == "AC":
player_config = get_ac_config(env_type="paper_review",
scoring_method=args.ac_scoring_method,
num_papers_per_area_chair=args.num_papers_per_area_chair,
global_settings=experiment_setting['global_settings'],
acceptance_rate=args.acceptance_rate,
**player_config)
player_config['model'] = args.model_name
player = AreaChair(data_dir=args.data_dir,
conference=args.conference,
args=args,
**player_config)
player_names.append(player.name)
elif role == "Author":
# Author requires no behavior customization.
# So we directly use the Player class
player_config = get_author_config()
player = Player(data_dir=args.data_dir,
conference=args.conference,
args=args,
**player_config)
player_names.append(player.name)
elif role == "Reviewer":
player_config = get_reviewer_player_config(reviewer_index=i + 1,
global_settings=experiment_setting['global_settings'],
**player_config)
player_config['model'] = args.model_name
player = Reviewer(data_dir=args.data_dir, conference=args.conference, **player_config)
player_names.append(player.name)
else:
raise NotImplementedError(f"Unknown role: {role}")
players.append(player)
env = PaperReview(player_names=player_names, paper_decision=paper_decision, paper_id=paper_id,
args=args, experiment_setting=experiment_setting)
arena = PaperReviewArena(players=players, environment=env, args=args)
arena.launch_cli(interactive=False)
logger.info("Done!")
if __name__ == "__main__":
project_setup()
main(parse_args())
|