evaluate_result/new-model.json

{
    "humaneval": {
        "pass@1": [
            [
                0,
                1.0
            ],
            [
                1,
                0.0
            ],
            [
                2,
                0.95
            ],
            [
                3,
                1.0
            ],
            [
                4,
                1.0
            ],
            [
                5,
                0.8999999999999999
            ],
            [
                6,
                0.20000000000000007
            ],
            [
                7,
                1.0
            ],
            [
                8,
                1.0
            ],
            [
                9,
                0.4999999999999999
            ],
            [
                10,
                0.050000000000000044
            ],
            [
                11,
                1.0
            ],
            [
                12,
                0.8999999999999999
            ],
            [
                13,
                1.0
            ],
            [
                14,
                0.7999999999999999
            ],
            [
                15,
                1.0
            ],
            [
                16,
                1.0
            ],
            [
                17,
                0.7
            ],
            [
                18,
                0.7
            ],
            [
                19,
                0.3500000000000001
            ],
            [
                20,
                0.4
            ],
            [
                21,
                0.85
            ],
            [
                22,
                1.0
            ],
            [
                23,
                1.0
            ],
            [
                24,
                0.8999999999999999
            ],
            [
                25,
                0.85
            ],
            [
                26,
                0.0
            ],
            [
                27,
                1.0
            ],
            [
                28,
                1.0
            ],
            [
                29,
                1.0
            ],
            [
                30,
                1.0
            ],
            [
                31,
                1.0
            ],
            [
                32,
                0.0
            ],
            [
                33,
                0.050000000000000044
            ],
            [
                34,
                1.0
            ],
            [
                35,
                1.0
            ],
            [
                36,
                0.5499999999999998
            ],
            [
                37,
                0.25
            ],
            [
                38,
                1.0
            ],
            [
                39,
                0.0
            ],
            [
                40,
                1.0
            ],
            [
                41,
                0.050000000000000044
            ],
            [
                42,
                1.0
            ],
            [
                43,
                1.0
            ],
            [
                44,
                1.0
            ],
            [
                45,
                1.0
            ],
            [
                46,
                0.4999999999999999
            ],
            [
                47,
                1.0
            ],
            [
                48,
                1.0
            ],
            [
                49,
                0.6000000000000001
            ],
            [
                50,
                1.0
            ],
            [
                51,
                0.8999999999999999
            ],
            [
                52,
                1.0
            ],
            [
                53,
                1.0
            ],
            [
                54,
                0.0
            ],
            [
                55,
                0.85
            ],
            [
                56,
                0.95
            ],
            [
                57,
                0.4
            ],
            [
                58,
                0.7999999999999999
            ],
            [
                59,
                0.44999999999999984
            ],
            [
                60,
                1.0
            ],
            [
                61,
                0.95
            ],
            [
                62,
                0.7
            ],
            [
                63,
                0.4
            ],
            [
                64,
                0.0
            ],
            [
                65,
                0.4
            ],
            [
                66,
                1.0
            ],
            [
                67,
                0.25
            ],
            [
                68,
                0.7999999999999999
            ],
            [
                69,
                0.7
            ],
            [
                70,
                0.25
            ],
            [
                71,
                0.85
            ],
            [
                72,
                0.5499999999999998
            ],
            [
                73,
                0.09999999999999998
            ],
            [
                74,
                1.0
            ],
            [
                75,
                0.0
            ],
            [
                76,
                0.15000000000000002
            ],
            [
                77,
                0.0
            ],
            [
                78,
                0.4999999999999999
            ],
            [
                79,
                0.050000000000000044
            ],
            [
                80,
                1.0
            ],
            [
                81,
                0.0
            ],
            [
                82,
                0.5499999999999998
            ],
            [
                83,
                0.0
            ],
            [
                84,
                0.050000000000000044
            ],
            [
                85,
                1.0
            ],
            [
                86,
                0.7999999999999999
            ],
            [
                87,
                0.5499999999999998
            ],
            [
                88,
                0.20000000000000007
            ],
            [
                89,
                0.8999999999999999
            ],
            [
                90,
                0.09999999999999998
            ],
            [
                91,
                0.15000000000000002
            ],
            [
                92,
                0.7999999999999999
            ],
            [
                93,
                0.050000000000000044
            ],
            [
                94,
                0.30000000000000004
            ],
            [
                95,
                0.050000000000000044
            ],
            [
                96,
                0.8999999999999999
            ],
            [
                97,
                1.0
            ],
            [
                98,
                0.8999999999999999
            ],
            [
                99,
                0.3500000000000001
            ],
            [
                100,
                0.15000000000000002
            ],
            [
                101,
                0.09999999999999998
            ],
            [
                102,
                0.7
            ],
            [
                103,
                0.5499999999999998
            ],
            [
                104,
                0.3500000000000001
            ],
            [
                105,
                0.20000000000000007
            ],
            [
                106,
                0.0
            ],
            [
                107,
                0.8999999999999999
            ],
            [
                108,
                0.0
            ],
            [
                109,
                0.15000000000000002
            ],
            [
                110,
                0.15000000000000002
            ],
            [
                111,
                0.6000000000000001
            ],
            [
                112,
                0.7999999999999999
            ],
            [
                113,
                0.7
            ],
            [
                114,
                1.0
            ],
            [
                115,
                0.050000000000000044
            ],
            [
                116,
                0.15000000000000002
            ],
            [
                117,
                0.65
            ],
            [
                118,
                0.0
            ],
            [
                119,
                0.09999999999999998
            ],
            [
                120,
                0.09999999999999998
            ],
            [
                121,
                0.5499999999999998
            ],
            [
                122,
                0.85
            ],
            [
                123,
                0.4999999999999999
            ],
            [
                124,
                0.050000000000000044
            ],
            [
                125,
                0.050000000000000044
            ],
            [
                126,
                0.050000000000000044
            ],
            [
                127,
                0.050000000000000044
            ],
            [
                128,
                0.4999999999999999
            ],
            [
                129,
                0.0
            ],
            [
                130,
                0.0
            ],
            [
                131,
                0.0
            ],
            [
                132,
                0.0
            ],
            [
                133,
                0.0
            ],
            [
                134,
                0.0
            ],
            [
                135,
                0.0
            ],
            [
                136,
                0.8999999999999999
            ],
            [
                137,
                0.0
            ],
            [
                138,
                0.0
            ],
            [
                139,
                0.09999999999999998
            ],
            [
                140,
                0.050000000000000044
            ],
            [
                141,
                0.65
            ],
            [
                142,
                0.25
            ],
            [
                143,
                0.15000000000000002
            ],
            [
                144,
                0.44999999999999984
            ],
            [
                145,
                0.0
            ],
            [
                146,
                0.20000000000000007
            ],
            [
                147,
                0.8999999999999999
            ],
            [
                148,
                0.7
            ],
            [
                149,
                0.5499999999999998
            ],
            [
                150,
                0.3500000000000001
            ],
            [
                151,
                1.0
            ],
            [
                152,
                1.0
            ],
            [
                153,
                0.75
            ],
            [
                154,
                0.5499999999999998
            ],
            [
                155,
                0.25
            ],
            [
                156,
                0.44999999999999984
            ],
            [
                157,
                1.0
            ],
            [
                158,
                0.8999999999999999
            ],
            [
                159,
                0.8999999999999999
            ],
            [
                160,
                0.15000000000000002
            ],
            [
                161,
                0.25
            ],
            [
                162,
                1.0
            ],
            [
                163,
                0.0
            ]
        ],
        "pass@10": [
            [
                0,
                1.0
            ],
            [
                1,
                0.0
            ],
            [
                2,
                1.0
            ],
            [
                3,
                1.0
            ],
            [
                4,
                1.0
            ],
            [
                5,
                1.0
            ],
            [
                6,
                0.956656346749226
            ],
            [
                7,
                1.0
            ],
            [
                8,
                1.0
            ],
            [
                9,
                0.9999945874558878
            ],
            [
                10,
                0.5
            ],
            [
                11,
                1.0
            ],
            [
                12,
                1.0
            ],
            [
                13,
                1.0
            ],
            [
                14,
                1.0
            ],
            [
                15,
                1.0
            ],
            [
                16,
                1.0
            ],
            [
                17,
                1.0
            ],
            [
                18,
                1.0
            ],
            [
                19,
                0.9984520123839009
            ],
            [
                20,
                0.9996427720885925
            ],
            [
                21,
                1.0
            ],
            [
                22,
                1.0
            ],
            [
                23,
                1.0
            ],
            [
                24,
                1.0
            ],
            [
                25,
                1.0
            ],
            [
                26,
                0.0
            ],
            [
                27,
                1.0
            ],
            [
                28,
                1.0
            ],
            [
                29,
                1.0
            ],
            [
                30,
                1.0
            ],
            [
                31,
                1.0
            ],
            [
                32,
                0.0
            ],
            [
                33,
                0.5
            ],
            [
                34,
                1.0
            ],
            [
                35,
                1.0
            ],
            [
                36,
                1.0
            ],
            [
                37,
                0.9837461300309598
            ],
            [
                38,
                1.0
            ],
            [
                39,
                0.0
            ],
            [
                40,
                1.0
            ],
            [
                41,
                0.5
            ],
            [
                42,
                1.0
            ],
            [
                43,
                1.0
            ],
            [
                44,
                1.0
            ],
            [
                45,
                1.0
            ],
            [
                46,
                0.9999945874558878
            ],
            [
                47,
                1.0
            ],
            [
                48,
                1.0
            ],
            [
                49,
                1.0
            ],
            [
                50,
                1.0
            ],
            [
                51,
                1.0
            ],
            [
                52,
                1.0
            ],
            [
                53,
                1.0
            ],
            [
                54,
                0.0
            ],
            [
                55,
                1.0
            ],
            [
                56,
                1.0
            ],
            [
                57,
                0.9996427720885925
            ],
            [
                58,
                1.0
            ],
            [
                59,
                0.9999404620147654
            ],
            [
                60,
                1.0
            ],
            [
                61,
                1.0
            ],
            [
                62,
                1.0
            ],
            [
                63,
                0.9996427720885925
            ],
            [
                64,
                0.0
            ],
            [
                65,
                0.9996427720885925
            ],
            [
                66,
                1.0
            ],
            [
                67,
                0.9837461300309598
            ],
            [
                68,
                1.0
            ],
            [
                69,
                1.0
            ],
            [
                70,
                0.9837461300309598
            ],
            [
                71,
                1.0
            ],
            [
                72,
                1.0
            ],
            [
                73,
                0.763157894736842
            ],
            [
                74,
                1.0
            ],
            [
                75,
                0.0
            ],
            [
                76,
                0.8947368421052632
            ],
            [
                77,
                0.0
            ],
            [
                78,
                0.9999945874558878
            ],
            [
                79,
                0.5
            ],
            [
                80,
                1.0
            ],
            [
                81,
                0.0
            ],
            [
                82,
                1.0
            ],
            [
                83,
                0.0
            ],
            [
                84,
                0.5
            ],
            [
                85,
                1.0
            ],
            [
                86,
                1.0
            ],
            [
                87,
                1.0
            ],
            [
                88,
                0.956656346749226
            ],
            [
                89,
                1.0
            ],
            [
                90,
                0.763157894736842
            ],
            [
                91,
                0.8947368421052632
            ],
            [
                92,
                1.0
            ],
            [
                93,
                0.5
            ],
            [
                94,
                0.9945820433436533
            ],
            [
                95,
                0.5
            ],
            [
                96,
                1.0
            ],
            [
                97,
                1.0
            ],
            [
                98,
                1.0
            ],
            [
                99,
                0.9984520123839009
            ],
            [
                100,
                0.8947368421052632
            ],
            [
                101,
                0.763157894736842
            ],
            [
                102,
                1.0
            ],
            [
                103,
                1.0
            ],
            [
                104,
                0.9984520123839009
            ],
            [
                105,
                0.956656346749226
            ],
            [
                106,
                0.0
            ],
            [
                107,
                1.0
            ],
            [
                108,
                0.0
            ],
            [
                109,
                0.8947368421052632
            ],
            [
                110,
                0.8947368421052632
            ],
            [
                111,
                1.0
            ],
            [
                112,
                1.0
            ],
            [
                113,
                1.0
            ],
            [
                114,
                1.0
            ],
            [
                115,
                0.5
            ],
            [
                116,
                0.8947368421052632
            ],
            [
                117,
                1.0
            ],
            [
                118,
                0.0
            ],
            [
                119,
                0.763157894736842
            ],
            [
                120,
                0.763157894736842
            ],
            [
                121,
                1.0
            ],
            [
                122,
                1.0
            ],
            [
                123,
                0.9999945874558878
            ],
            [
                124,
                0.5
            ],
            [
                125,
                0.5
            ],
            [
                126,
                0.5
            ],
            [
                127,
                0.5
            ],
            [
                128,
                0.9999945874558878
            ],
            [
                129,
                0.0
            ],
            [
                130,
                0.0
            ],
            [
                131,
                0.0
            ],
            [
                132,
                0.0
            ],
            [
                133,
                0.0
            ],
            [
                134,
                0.0
            ],
            [
                135,
                0.0
            ],
            [
                136,
                1.0
            ],
            [
                137,
                0.0
            ],
            [
                138,
                0.0
            ],
            [
                139,
                0.763157894736842
            ],
            [
                140,
                0.5
            ],
            [
                141,
                1.0
            ],
            [
                142,
                0.9837461300309598
            ],
            [
                143,
                0.8947368421052632
            ],
            [
                144,
                0.9999404620147654
            ],
            [
                145,
                0.0
            ],
            [
                146,
                0.956656346749226
            ],
            [
                147,
                1.0
            ],
            [
                148,
                1.0
            ],
            [
                149,
                1.0
            ],
            [
                150,
                0.9984520123839009
            ],
            [
                151,
                1.0
            ],
            [
                152,
                1.0
            ],
            [
                153,
                1.0
            ],
            [
                154,
                1.0
            ],
            [
                155,
                0.9837461300309598
            ],
            [
                156,
                0.9999404620147654
            ],
            [
                157,
                1.0
            ],
            [
                158,
                1.0
            ],
            [
                159,
                1.0
            ],
            [
                160,
                0.8947368421052632
            ],
            [
                161,
                0.9837461300309598
            ],
            [
                162,
                1.0
            ],
            [
                163,
                0.0
            ]
        ]
    },
    "config": {
        "prefix": "",
        "do_sample": true,
        "temperature": 0.2,
        "top_k": 0,
        "top_p": 0.95,
        "n_samples": 20,
        "eos": "<|endoftext|>",
        "seed": 0,
        "model": "1_java_codegemma-2b_multiple-java",
        "modeltype": "causal",
        "peft_model": null,
        "revision": null,
        "use_auth_token": false,
        "trust_remote_code": false,
        "tasks": "humaneval",
        "instruction_tokens": null,
        "batch_size": 1,
        "max_length_generation": 512,
        "precision": "fp32",
        "load_in_8bit": false,
        "load_in_4bit": false,
        "left_padding": false,
        "limit": null,
        "limit_start": 0,
        "save_every_k_tasks": -1,
        "postprocess": true,
        "allow_code_execution": true,
        "generation_only": false,
        "load_generations_path": "1_codegemma-7b-it_generations_humaneval_codegemma-7b-it_humaneval.json",
        "load_data_path": null,
        "metric_output_path": "hhhhh.json",
        "save_generations": false,
        "load_generations_intermediate_paths": null,
        "save_generations_path": "generations.json",
        "save_references": false,
        "save_references_path": "references.json",
        "prompt": "prompt",
        "max_memory_per_gpu": null,
        "check_references": false
    }
}