diff --git "a/reduced_ppl_average.json" "b/reduced_ppl_average.json" new file mode 100644--- /dev/null +++ "b/reduced_ppl_average.json" @@ -0,0 +1,33680 @@ +{ + "original_ppl": 3.5625619888305664, + "reduced_ppl": [ + { + "bit": 1, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 31421.43359375 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 31576.380859375 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 30524.94921875 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 38853.6171875 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 32848.5859375 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 33773.57421875 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 37451.27734375 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 31632.953125 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 32005.0078125 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 36216.72265625 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 36035.16015625 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 34296.2421875 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 32313.29296875 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 32333.88671875 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 32259.041015625 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 31792.759765625 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 44054.37890625 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 31784.849609375 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 31832.140625 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 32830.453125 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 32508.609375 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 31262.98828125 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 31404.08984375 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 32538.015625 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 41311.12890625 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 32610.302734375 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 33147.16015625 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 31966.96875 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 31925.779296875 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 31874.759765625 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 31744.556640625 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 31513.0234375 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 31650.91015625 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 32102.216796875 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 32374.3984375 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 31881.537109375 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 31439.541015625 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 32424.671875 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 27770.126953125 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 32291.421875 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 32135.296875 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 32209.271484375 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 31742.83203125 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 31448.65625 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 33373.63671875 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 40139.296875 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 32058.283203125 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 33473.91796875 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 31896.8984375 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 31886.46484375 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 31778.119140625 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 31245.669921875 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 33631.03125 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 30932.814453125 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 31485.267578125 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 30746.52734375 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 31354.14453125 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 31454.8359375 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 29345.759765625 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 30430.86328125 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 30892.78125 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 31128.423828125 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 30723.83984375 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 31430.064453125 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 31564.818359375 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 30183.3671875 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 32539.130859375 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 30958.4921875 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 31154.115234375 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 31346.73046875 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 31462.513671875 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 31361.619140625 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 31554.494140625 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 30446.39453125 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 30995.595703125 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 31855.431640625 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 31224.609375 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 31430.90625 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 31251.09375 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 31104.86328125 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 32328.984375 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 32006.228515625 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 31758.8828125 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 31184.43359375 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 31678.75 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 31569.333984375 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 31794.791015625 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 31437.501953125 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 30908.400390625 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 30982.7109375 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 31495.900390625 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 31724.0390625 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 31597.345703125 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 31026.9453125 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 32273.53515625 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 31209.8125 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 31348.046875 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 30287.662109375 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 31041.98046875 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 31095.13671875 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 26738.25390625 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 30387.625 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 30587.98046875 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 30028.068359375 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 30052.8515625 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 30921.4921875 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 30704.44921875 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 29333.728515625 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 28096.615234375 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 31023.158203125 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 31160.919921875 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 31296.875 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 31033.306640625 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 31460.146484375 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 29508.5859375 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 30972.66796875 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 30671.259765625 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 30980.642578125 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 30315.6640625 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 31444.638671875 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 31318.58203125 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 34290.02734375 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 28223.048828125 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 31243.642578125 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 30575.236328125 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 30060.90625 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 31434.68359375 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 31435.79296875 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 31972.3046875 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 31593.669921875 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 31041.921875 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 31630.69140625 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 31097.53515625 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 31448.384765625 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 31488.69140625 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 33665.75 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 31033.69140625 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 31307.412109375 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 30959.376953125 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 30699.32421875 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 31401.603515625 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 31427.12890625 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 33702.2421875 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 30665.205078125 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 31253.65625 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 31648.67578125 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 30935.501953125 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 31383.40234375 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 31461.07421875 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 30499.517578125 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 32961.3671875 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 31627.9765625 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 30592.38671875 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 31533.919921875 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 31187.19921875 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 31095.283203125 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 31673.916015625 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 32358.072265625 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 31836.298828125 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 30909.990234375 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 31488.630859375 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 31530.7890625 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 31652.115234375 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 33405.29296875 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 31515.7890625 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 31307.29296875 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 31079.541015625 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 31206.419921875 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 31483.044921875 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 31541.13671875 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 32348.134765625 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 31566.71484375 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 31704.291015625 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 30749.986328125 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 31015.673828125 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 31659.30078125 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 31754.9140625 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 32583.87890625 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 31721.740234375 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 32082.017578125 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 31150.96484375 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 30791.306640625 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 31907.849609375 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 31762.91015625 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 31347.685546875 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 31927.576171875 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 32521.107421875 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 31050.033203125 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 31457.65625 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 31452.28515625 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 31320.134765625 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 31864.76171875 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 32645.15234375 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 30967.0546875 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 31032.626953125 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 31582.5234375 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 31304.27734375 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 31228.896484375 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 30254.8671875 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 31685.69921875 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 31507.794921875 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 31714.84375 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 30664.884765625 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 31689.0234375 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 31554.734375 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 30064.345703125 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 32519.4921875 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 31672.4375 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 31225.4140625 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 31492.7734375 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 31306.458984375 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 31539.78125 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 31389.388671875 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 32554.091796875 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 31224.37109375 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 32070.26953125 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 31172.689453125 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 31011.443359375 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 30710.892578125 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 29564.671875 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 33247.140625 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 31263.1953125 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 31349.66015625 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 32055.927734375 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 31484.576171875 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 31392.083984375 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 31633.345703125 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 30400.841796875 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 31428.748046875 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 31309.62109375 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 31428.208984375 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 31613.681640625 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 31501.09375 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 29624.587890625 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 31483.07421875 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 30736.853515625 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 31508.125 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 31700.630859375 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 31461.13671875 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 31227.736328125 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 32171.05078125 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 31966.0546875 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 31572.103515625 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 30871.341796875 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 31306.755859375 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 31281.1484375 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 31645.958984375 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 32219.072265625 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 33282.54296875 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 31077.615234375 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 31480.58203125 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 31796.458984375 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 31172.5703125 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 31304.63671875 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 31114.94921875 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 30095.701171875 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 31156.9375 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 31636.96484375 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 31831.138671875 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 31766.361328125 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 31556.78125 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 32155.498046875 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 31262.2421875 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 31362.369140625 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 31575.0546875 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 31336.6875 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 31581.2890625 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 31559.33984375 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 30515.8671875 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 31743.287109375 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 31422.544921875 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 31456.875 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 31307.591796875 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 31210.67578125 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 31263.732421875 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 32157.30859375 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 32371.466796875 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 31440.5 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 31014.697265625 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 31676.6953125 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 31298.724609375 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 31368.619140625 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 32018.013671875 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 30966.34765625 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 31250.826171875 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 31351.095703125 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 31533.28515625 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 31380.109375 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 31502.5078125 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 30771.138671875 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 31424.43359375 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 31382.11328125 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 32101.8203125 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 31011.798828125 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 31504.970703125 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 31350.318359375 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 31543.93359375 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 31115.783203125 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 31282.611328125 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 31130.1484375 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 31357.3125 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 31427.310546875 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 31403.1328125 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 31082.1484375 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 31768.908203125 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 31205.794921875 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 31064.013671875 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 31161.037109375 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 31523.60546875 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 31276.138671875 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 31814.75 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 31282.880859375 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 31585.505859375 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 31131.6328125 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 31488.630859375 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 31336.5078125 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 31379.8984375 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 30919.189453125 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 31113.94140625 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 31849.751953125 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 31303.232421875 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 31484.697265625 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 31459.21484375 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 31460.32421875 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 31703.71484375 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 31188.716796875 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 31367.9609375 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 31295.29296875 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 31189.193359375 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 31422.693359375 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 31414.423828125 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 31350.138671875 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 31429.107421875 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 31989.751953125 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 31773.08984375 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 31442.837890625 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 31455.615234375 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 31226.6640625 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 31865.88671875 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 31428.689453125 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 31071.30078125 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 31594.99609375 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 31904.3203125 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 31605.0 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 31221.06640625 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 31248.111328125 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 30711.654296875 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 31533.767578125 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 31077.79296875 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 31519.515625 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 31401.90234375 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 31639.380859375 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 31282.40234375 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 31475.1796875 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 31238.609375 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 31694.978515625 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 31485.02734375 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 31569.697265625 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 31282.521484375 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 31854.033203125 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 30736.853515625 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 31756.095703125 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 32085.75 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 31613.322265625 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 31400.587890625 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 31506.11328125 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 31379.15234375 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 31310.279296875 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 31499.0234375 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 30960.587890625 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 31392.681640625 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 31238.876953125 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 31165.5546875 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 30527.076171875 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 31568.19140625 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 31511.28125 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 31416.8203125 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 31218.86328125 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 31312.21875 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 31403.16015625 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 30925.9140625 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 30919.072265625 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 31734.751953125 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 31868.498046875 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 31907.607421875 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 31630.267578125 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 31712.54296875 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 32128.189453125 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 30755.03125 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 31279.03125 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 31134.896484375 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 31426.6796875 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 31389.748046875 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 31210.705078125 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 31863.697265625 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 31253.865234375 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 31347.595703125 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 31820.7578125 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 31695.19140625 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 31262.6875 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 31410.91796875 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 31385.6171875 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 31329.75390625 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 31208.7421875 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 31950.9375 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 31013.337890625 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 31534.912109375 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 31570.8984375 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 30836.62109375 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 32091.197265625 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 31584.572265625 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 31230.86328125 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 31224.04296875 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 31384.478515625 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 31403.939453125 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 31476.80078125 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 32164.580078125 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 31111.30078125 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 30964.455078125 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 32070.02734375 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 31508.755859375 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 31604.427734375 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 31045.029296875 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 30740.4296875 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 31398.0703125 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 31561.869140625 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 31257.232421875 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 31429.556640625 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 31422.994140625 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 31075.984375 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 30612.349609375 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 31225.294921875 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 31505.30078125 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 31558.6171875 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 31411.66796875 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 31368.7109375 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 31210.736328125 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 30833.474609375 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 30946.41796875 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 30757.5546875 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 31132.046875 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 31195.947265625 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 31203.59375 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 32642.4140625 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 30972.251953125 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 31104.685546875 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 31222.583984375 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 31859.77734375 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 31409.271484375 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 31604.15625 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 31404.419921875 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 30858.330078125 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 31459.904296875 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 31375.41015625 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 31733.05859375 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 31393.9375 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 31350.765625 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 31267.669921875 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 32429.0625 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 31690.29296875 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 31311.4140625 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 31760.91015625 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 31459.513671875 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 31484.845703125 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 31254.818359375 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 30641.908203125 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 31450.515625 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 31185.951171875 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 31489.23046875 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 31337.943359375 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 31438.220703125 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 32128.86328125 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 30532.63671875 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 31667.12109375 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 31306.3671875 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 32379.24609375 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 31414.841796875 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 31401.544921875 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 33018.75390625 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 31601.505859375 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 30970.541015625 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 30980.525390625 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 30676.732421875 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 31692.318359375 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 31457.65625 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 31401.125 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 29547.560546875 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 31355.28125 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 30998.462890625 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 31194.279296875 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 31903.529296875 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 30491.404296875 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 29556.0703125 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 32232.31640625 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 30988.119140625 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 32906.0234375 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 31389.08984375 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 31117.76953125 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 30615.005859375 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 32235.484375 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 33410.26171875 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 31104.685546875 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 30342.015625 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 31967.974609375 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 30148.384765625 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 32071.33984375 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 31878.892578125 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 33338.01171875 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 30684.89453125 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 31766.96875 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 30970.0078125 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 31291.98046875 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 31670.68359375 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 30608.525390625 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 31190.708984375 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 30610.9765625 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 32061.064453125 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 31591.19921875 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 31228.80859375 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 31575.9296875 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 31548.77734375 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 30700.525390625 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 31188.15234375 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 33632.88671875 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 32066.783203125 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 31075.8359375 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 31738.685546875 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 29586.275390625 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 32448.2734375 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 28725.1875 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 32619.10546875 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 32109.53515625 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 31527.634765625 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 31761.3046875 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 31060.310546875 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 28689.5703125 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 32480.44140625 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 32555.271484375 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 32503.310546875 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 31459.81640625 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 31585.865234375 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 30666.755859375 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 30155.28515625 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 29916.765625 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 33321.1328125 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 32821.75 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 32087.400390625 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 32064.21484375 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 30550.607421875 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 31552.658203125 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 27280.130859375 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 36173.78125 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 33956.5625 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 31714.904296875 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 30999.556640625 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 31853.73046875 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 32167.646484375 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 28544.951171875 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 36654.6796875 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 28946.39453125 + } + ] + }, + { + "bit": 2, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 13.645581245422363 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 13.635476112365723 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 13.597652435302734 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 13.680755615234375 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 13.642777442932129 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 13.394048690795898 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 13.564644813537598 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 13.433846473693848 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 13.399696350097656 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 13.47042179107666 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 13.33731460571289 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 13.4304780960083 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 13.397647857666016 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 13.314291954040527 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 13.547911643981934 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 13.41981315612793 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 13.611660957336426 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 13.408958435058594 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 13.450536727905273 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 13.755903244018555 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 13.515820503234863 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 13.6262788772583 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 13.80778980255127 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 13.47126293182373 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 13.498884201049805 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 13.534930229187012 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 13.507692337036133 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 13.550167083740234 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 13.608172416687012 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 13.621481895446777 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 13.599678993225098 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 13.634679794311523 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 13.571560859680176 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 13.575858116149902 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 13.557266235351562 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 13.613987922668457 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 13.616393089294434 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 13.585844039916992 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 13.576133728027344 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 13.563948631286621 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 13.582453727722168 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 13.581695556640625 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 13.605818748474121 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 13.599350929260254 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 13.587583541870117 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 13.561443328857422 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 13.583255767822266 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 13.496274948120117 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 13.586838722229004 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 13.632124900817871 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 13.622517585754395 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 13.589715003967285 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 13.665265083312988 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 13.54382610321045 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 13.534635543823242 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 13.541533470153809 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 13.614861488342285 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 13.617856979370117 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 13.619916915893555 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 13.62283992767334 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 13.573397636413574 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 13.583596229553223 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 13.590635299682617 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 13.600171089172363 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 13.593056678771973 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 13.592512130737305 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 13.616129875183105 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 13.59862232208252 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 13.57503604888916 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 13.568086624145508 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 13.655845642089844 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 13.640421867370605 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 13.566289901733398 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 13.588808059692383 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 13.58224868774414 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 13.582802772521973 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 13.58047103881836 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 13.599318504333496 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 13.616929054260254 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 13.5665922164917 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 13.618494033813477 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 13.583809852600098 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 13.608354568481445 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 13.605876922607422 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 13.6194486618042 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 13.624872207641602 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 13.524397850036621 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 13.565045356750488 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 13.617286682128906 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 13.635200500488281 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 13.660065650939941 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 13.563467025756836 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 13.565898895263672 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 13.553911209106445 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 13.592324256896973 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 13.564708709716797 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 13.612443923950195 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 13.593613624572754 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 13.597879409790039 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 13.604134559631348 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 13.574699401855469 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 13.55550765991211 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 13.53783130645752 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 13.577738761901855 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 13.558798789978027 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 13.616497039794922 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 13.610711097717285 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 13.60762882232666 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 13.552186012268066 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 13.626432418823242 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 13.63128662109375 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 13.590421676635742 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 13.598845481872559 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 13.621384620666504 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 13.596368789672852 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 13.567843437194824 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 13.605247497558594 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 13.476011276245117 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 13.521073341369629 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 13.639397621154785 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 13.598145484924316 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 13.612910270690918 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 13.498391151428223 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 13.728271484375 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 13.73729419708252 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 13.642128944396973 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 13.612810134887695 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 13.633255958557129 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 13.577774047851562 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 13.542211532592773 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 13.563518524169922 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 13.466221809387207 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 13.490156173706055 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 13.623908042907715 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 13.598281860351562 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 13.668317794799805 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 13.525568008422852 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 13.586576461791992 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 13.553268432617188 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 13.494095802307129 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 13.64867877960205 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 13.625123023986816 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 13.583162307739258 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 13.611038208007812 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 13.612141609191895 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 13.59817123413086 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 13.518281936645508 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 13.588713645935059 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 13.596816062927246 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 13.489093780517578 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 13.579943656921387 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 13.567769050598145 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 13.6852388381958 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 13.606064796447754 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 13.560317993164062 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 13.585226058959961 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 13.518733024597168 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 13.450216293334961 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 13.528932571411133 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 13.749955177307129 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 13.621850967407227 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 13.549452781677246 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 13.54078483581543 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 13.560479164123535 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 13.408599853515625 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 13.472509384155273 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 13.738264083862305 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 13.648258209228516 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 13.665037155151367 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 13.639803886413574 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 13.483826637268066 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 13.335184097290039 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 13.446946144104004 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 13.727295875549316 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 13.606849670410156 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 13.617546081542969 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 13.596329689025879 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 13.592044830322266 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 13.362870216369629 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 13.269471168518066 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 13.554929733276367 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 13.411771774291992 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 13.503297805786133 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 13.5665922164917 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 13.478864669799805 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 13.584794998168945 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 13.47333812713623 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 13.554089546203613 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 13.561853408813477 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 13.5211181640625 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 13.565436363220215 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 13.579341888427734 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 13.510333061218262 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 13.556649208068848 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 13.578810691833496 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 13.504150390625 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 13.650347709655762 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 13.558755874633789 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 13.632304191589355 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 13.588824272155762 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 13.509998321533203 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 13.503973007202148 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 13.530357360839844 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 13.521608352661133 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 13.49836540222168 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 13.621027946472168 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 13.447254180908203 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 13.402999877929688 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 13.633772850036621 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 13.562483787536621 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 13.655731201171875 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 13.602334976196289 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 13.534652709960938 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 13.505208969116211 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 13.493790626525879 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 13.547353744506836 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 13.480253219604492 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 13.549277305603027 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 13.56069278717041 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 13.551210403442383 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 13.581934928894043 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 13.595986366271973 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 13.593175888061523 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 13.607141494750977 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 13.585772514343262 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 13.587574005126953 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 13.65427017211914 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 13.570143699645996 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 13.541049003601074 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 13.530238151550293 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 13.551222801208496 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 13.59394359588623 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 13.619220733642578 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 13.623229026794434 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 13.502930641174316 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 13.582977294921875 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 13.543192863464355 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 13.57805347442627 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 13.582022666931152 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 13.556742668151855 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 13.600862503051758 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 13.467143058776855 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 13.58840274810791 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 13.617786407470703 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 13.543490409851074 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 13.61699390411377 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 13.599818229675293 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 13.612332344055176 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 13.556496620178223 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 13.585591316223145 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 13.5930757522583 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 13.597538948059082 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 13.603984832763672 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 13.615897178649902 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 13.593756675720215 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 13.58084774017334 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 13.581915855407715 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 13.601305961608887 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 13.586835861206055 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 13.582883834838867 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 13.606039047241211 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 13.629778861999512 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 13.573586463928223 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 13.576828956604004 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 13.609244346618652 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 13.600214004516602 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 13.646709442138672 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 13.615894317626953 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 13.6046724319458 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 13.59933853149414 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 13.580163955688477 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 13.595182418823242 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 13.600265502929688 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 13.595428466796875 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 13.600530624389648 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 13.587777137756348 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 13.594002723693848 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 13.588075637817383 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 13.605743408203125 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 13.59485149383545 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 13.601679801940918 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 13.612687110900879 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 13.617390632629395 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 13.614819526672363 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 13.586109161376953 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 13.586368560791016 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 13.582494735717773 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 13.60535717010498 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 13.611936569213867 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 13.600885391235352 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 13.6038818359375 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 13.603826522827148 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 13.615248680114746 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 13.597908020019531 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 13.605694770812988 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 13.610649108886719 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 13.61190128326416 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 13.602314949035645 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 13.596501350402832 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 13.611885070800781 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 13.600135803222656 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 13.605412483215332 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 13.608659744262695 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 13.607342720031738 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 13.594329833984375 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 13.596673965454102 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 13.599130630493164 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 13.596879959106445 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 13.604657173156738 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 13.608014106750488 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 13.619913101196289 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 13.578752517700195 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 13.59168529510498 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 13.59862232208252 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 13.595436096191406 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 13.600881576538086 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 13.609896659851074 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 13.607881546020508 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 13.596651077270508 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 13.607661247253418 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 13.612787246704102 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 13.602428436279297 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 13.61508846282959 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 13.614595413208008 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 13.612191200256348 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 13.6124267578125 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 13.590256690979004 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 13.595785140991211 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 13.595762252807617 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 13.614117622375488 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 13.616971969604492 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 13.614286422729492 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 13.624275207519531 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 13.598252296447754 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 13.602009773254395 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 13.607488632202148 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 13.604657173156738 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 13.614896774291992 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 13.608955383300781 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 13.620686531066895 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 13.598160743713379 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 13.606161117553711 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 13.592362403869629 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 13.61372184753418 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 13.616832733154297 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 13.625054359436035 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 13.622352600097656 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 13.598553657531738 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 13.61463737487793 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 13.594573020935059 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 13.615180015563965 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 13.617286682128906 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 13.616305351257324 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 13.613631248474121 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 13.598326683044434 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 13.606206893920898 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 13.60067367553711 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 13.615851402282715 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 13.617257118225098 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 13.617318153381348 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 13.61495590209961 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 13.592994689941406 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 13.60081672668457 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 13.601744651794434 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 13.613020896911621 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 13.618013381958008 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 13.596400260925293 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 13.603874206542969 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 13.596537590026855 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 13.608406066894531 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 13.60534381866455 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 13.620397567749023 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 13.620559692382812 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 13.617169380187988 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 13.616040229797363 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 13.59860610961914 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 13.60096549987793 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 13.602730751037598 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 13.591004371643066 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 13.602782249450684 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 13.61425495147705 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 13.598421096801758 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 13.601815223693848 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 13.602493286132812 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 13.60611629486084 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 13.59799575805664 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 13.610353469848633 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 13.6267728805542 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 13.616935729980469 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 13.603141784667969 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 13.606382369995117 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 13.598650932312012 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 13.618816375732422 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 13.620007514953613 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 13.623384475708008 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 13.609785079956055 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 13.601757049560547 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 13.600687026977539 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 13.604413032531738 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 13.609214782714844 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 13.617196083068848 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 13.606888771057129 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 13.610989570617676 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 13.603874206542969 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 13.608640670776367 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 13.606083869934082 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 13.616065979003906 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 13.61586856842041 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 13.617728233337402 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 13.61373519897461 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 13.600735664367676 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 13.60423755645752 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 13.607040405273438 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 13.614442825317383 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 13.61287784576416 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 13.61423110961914 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 13.612306594848633 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 13.600035667419434 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 13.611936569213867 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 13.603103637695312 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 13.624557495117188 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 13.630857467651367 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 13.62211799621582 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 13.626093864440918 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 13.606441497802734 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 13.610615730285645 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 13.602837562561035 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 13.615370750427246 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 13.618592262268066 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 13.61533260345459 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 13.616942405700684 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 13.60394287109375 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 13.606926918029785 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 13.608903884887695 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 13.615612030029297 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 13.61676025390625 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 13.616580963134766 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 13.614604949951172 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 13.604558944702148 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 13.614241600036621 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 13.602127075195312 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 13.602548599243164 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 13.606816291809082 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 13.607556343078613 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 13.613605499267578 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 13.597387313842773 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 13.608659744262695 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 13.605331420898438 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 13.617825508117676 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 13.62468433380127 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 13.622088432312012 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 13.616942405700684 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 13.608546257019043 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 13.611355781555176 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 13.601462364196777 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 13.6127347946167 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 13.615374565124512 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 13.615978240966797 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 13.617059707641602 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 13.605863571166992 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 13.607436180114746 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 13.610795021057129 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 13.613543510437012 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 13.612702369689941 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 13.611833572387695 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 13.625255584716797 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 13.60438346862793 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 13.611385345458984 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 13.612034797668457 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 13.613404273986816 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 13.61469554901123 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 13.61844253540039 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 13.620205879211426 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 13.610353469848633 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 13.612163543701172 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 13.611235618591309 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 13.62614631652832 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 13.630922317504883 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 13.625665664672852 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 13.619579315185547 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 13.613153457641602 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 13.625418663024902 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 13.628933906555176 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 13.59472942352295 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 13.613404273986816 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 13.60843276977539 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 13.608530044555664 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 13.601705551147461 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 13.621147155761719 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 13.619010925292969 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 13.61280345916748 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 13.626744270324707 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 13.604257583618164 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 13.61495590209961 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 13.612565994262695 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 13.642942428588867 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 13.615199089050293 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 13.716829299926758 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 13.621793746948242 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 13.599306106567383 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 13.611451148986816 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 13.616825103759766 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 13.617903709411621 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 13.601851463317871 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 13.662010192871094 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 13.66174030303955 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 13.622880935668945 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 13.628094673156738 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 13.602529525756836 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 13.610516548156738 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 13.617714881896973 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 13.610129356384277 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 13.613774299621582 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 13.636090278625488 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 13.606475830078125 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 13.629631996154785 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 13.621663093566895 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 13.611404418945312 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 13.60051155090332 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 13.630890846252441 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 13.608468055725098 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 13.61017894744873 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 13.58769702911377 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 13.614222526550293 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 13.613119125366211 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 13.824197769165039 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 13.701529502868652 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 13.611719131469727 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 13.613540649414062 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 13.580357551574707 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 13.671613693237305 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 13.638106346130371 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 13.593749046325684 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 13.604915618896484 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 13.625243186950684 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 13.607171058654785 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 13.593425750732422 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 13.64990234375 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 13.629239082336426 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 13.660405158996582 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 13.638948440551758 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 13.651418685913086 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 13.652804374694824 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 13.576589584350586 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 13.65953254699707 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 13.629287719726562 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 13.60633659362793 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 13.616598129272461 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 13.629732131958008 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 13.6338472366333 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 13.57752799987793 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 13.637404441833496 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 13.585888862609863 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 13.649458885192871 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 13.630759239196777 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 13.638663291931152 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 13.622861862182617 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 13.449203491210938 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 13.430547714233398 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 13.192337036132812 + } + ] + }, + { + "bit": 3, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 6.685004234313965 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 6.68459939956665 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 6.720666885375977 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 6.716915130615234 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 6.684933185577393 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 6.682521820068359 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 6.727982521057129 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 6.6940155029296875 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 6.682474613189697 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 6.707605361938477 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 6.729203224182129 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 6.6805267333984375 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 6.677731990814209 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 6.676042556762695 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 6.681649684906006 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 6.683011054992676 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 6.68275785446167 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 6.68345832824707 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 6.680637836456299 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 6.678243637084961 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 6.682172775268555 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 6.677589416503906 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 6.681847095489502 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 6.677261829376221 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 6.680530071258545 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 6.680121898651123 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 6.681399345397949 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 6.680069923400879 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 6.684310436248779 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 6.686277866363525 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 6.6845831871032715 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 6.6854777336120605 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 6.680107593536377 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 6.6798577308654785 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 6.679494857788086 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 6.683598041534424 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 6.683909893035889 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 6.68071174621582 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 6.683712959289551 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 6.680901050567627 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 6.681782245635986 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 6.680836200714111 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 6.682905197143555 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 6.683887481689453 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 6.681684970855713 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 6.684220314025879 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 6.680953025817871 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 6.683218002319336 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 6.682073593139648 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 6.685110092163086 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 6.6846208572387695 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 6.683614253997803 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 6.684555530548096 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 6.681842803955078 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 6.679967880249023 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 6.682723045349121 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 6.6854939460754395 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 6.6843061447143555 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 6.68409538269043 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 6.687803745269775 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 6.683951377868652 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 6.684490203857422 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 6.685387134552002 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 6.684320449829102 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 6.6844353675842285 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 6.679609298706055 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 6.682836532592773 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 6.684939384460449 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 6.686556816101074 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 6.683668613433838 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 6.683927059173584 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 6.684550762176514 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 6.684586048126221 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 6.684432029724121 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 6.682219505310059 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 6.685119152069092 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 6.681954383850098 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 6.682491302490234 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 6.682757377624512 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 6.685492038726807 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 6.684089660644531 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 6.683465480804443 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 6.679781913757324 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 6.6837639808654785 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 6.685671806335449 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 6.688955783843994 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 6.685173034667969 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 6.686036109924316 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 6.68615198135376 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 6.686722755432129 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 6.684033393859863 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 6.678061008453369 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 6.676022529602051 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 6.681747913360596 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 6.6801652908325195 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 6.690011978149414 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 6.68825626373291 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 6.684220314025879 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 6.683241844177246 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 6.681167125701904 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 6.681208610534668 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 6.679689884185791 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 6.685837745666504 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 6.685413837432861 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 6.68445348739624 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 6.6824750900268555 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 6.6816840171813965 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 6.682529449462891 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 6.68487024307251 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 6.687506198883057 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 6.682740211486816 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 6.683629989624023 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 6.684062480926514 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 6.683710098266602 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 6.684019088745117 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 6.681426048278809 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 6.686732292175293 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 6.681897163391113 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 6.680540561676025 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 6.682492733001709 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 6.68079137802124 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 6.676613807678223 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 6.676273345947266 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 6.692087173461914 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 6.684786319732666 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 6.6863508224487305 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 6.681793212890625 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 6.682941913604736 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 6.673989295959473 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 6.67391300201416 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 6.685483932495117 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 6.68815279006958 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 6.6757893562316895 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 6.683901309967041 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 6.6839165687561035 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 6.679954528808594 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 6.67725133895874 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 6.676361560821533 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 6.678177833557129 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 6.673962116241455 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 6.685821056365967 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 6.684691905975342 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 6.6797566413879395 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 6.683497905731201 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 6.682670593261719 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 6.6861090660095215 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 6.679214954376221 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 6.681987762451172 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 6.681858539581299 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 6.6717915534973145 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 6.678791046142578 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 6.677117824554443 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 6.682623386383057 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 6.678470611572266 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 6.678143501281738 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 6.679349899291992 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 6.686773777008057 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 6.672204971313477 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 6.6775054931640625 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 6.689705848693848 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 6.678304672241211 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 6.681610107421875 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 6.6816558837890625 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 6.666962146759033 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 6.675348281860352 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 6.672245025634766 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 6.693765163421631 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 6.680354118347168 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 6.683504581451416 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 6.684331893920898 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 6.678934574127197 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 6.685558795928955 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 6.677177906036377 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 6.685088157653809 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 6.680749416351318 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 6.6855998039245605 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 6.684093475341797 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 6.685333728790283 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 6.676408290863037 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 6.668050289154053 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 6.675105571746826 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 6.672253131866455 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 6.685774803161621 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 6.680861473083496 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 6.68303108215332 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 6.679785251617432 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 6.66909122467041 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 6.679225921630859 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 6.673841953277588 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 6.6776862144470215 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 6.679475784301758 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 6.68949556350708 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 6.6797590255737305 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 6.680171966552734 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 6.68041467666626 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 6.6732659339904785 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 6.680105686187744 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 6.681463718414307 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 6.6790876388549805 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 6.683902263641357 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 6.681116580963135 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 6.684545040130615 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 6.678264141082764 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 6.676321029663086 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 6.678332328796387 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 6.686545372009277 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 6.683432579040527 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 6.672232151031494 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 6.6833086013793945 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 6.673824787139893 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 6.679004669189453 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 6.684121608734131 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 6.684638023376465 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 6.682651519775391 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 6.682837009429932 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 6.678023338317871 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 6.671217441558838 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 6.6780171394348145 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 6.677274703979492 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 6.676057815551758 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 6.677341938018799 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 6.6805877685546875 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 6.683660984039307 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 6.681278228759766 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 6.68294095993042 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 6.68562126159668 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 6.68752384185791 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 6.683947563171387 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 6.682379722595215 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 6.686316013336182 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 6.675583362579346 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 6.685005187988281 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 6.6847028732299805 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 6.6775712966918945 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 6.68943977355957 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 6.691118240356445 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 6.689350605010986 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 6.689242362976074 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 6.7004075050354 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 6.697800636291504 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 6.693077087402344 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 6.690364837646484 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 6.663458824157715 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 6.66433048248291 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 6.690593719482422 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 6.684332847595215 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 6.682257652282715 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 6.680144786834717 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 6.682403087615967 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 6.683035373687744 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 6.680482387542725 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 6.6819071769714355 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 6.681036949157715 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 6.684055805206299 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 6.67901086807251 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 6.679704666137695 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 6.683383464813232 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 6.683270454406738 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 6.6825385093688965 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 6.682669162750244 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 6.685858726501465 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 6.683200836181641 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 6.676484107971191 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 6.680081844329834 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 6.684781074523926 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 6.680549621582031 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 6.683767318725586 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 6.683724403381348 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 6.681384086608887 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 6.685715198516846 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 6.684271812438965 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 6.683013439178467 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 6.678749084472656 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 6.681872844696045 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 6.68310546875 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 6.6809515953063965 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 6.681695938110352 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 6.680680751800537 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 6.681424140930176 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 6.678310394287109 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 6.6847333908081055 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 6.683061599731445 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 6.6861467361450195 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 6.680845260620117 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 6.679870128631592 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 6.681001663208008 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 6.67796516418457 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 6.682969570159912 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 6.68363618850708 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 6.681868553161621 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 6.684538841247559 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 6.6799492835998535 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 6.682034015655518 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 6.684535503387451 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 6.684140205383301 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 6.684924602508545 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 6.6829681396484375 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 6.684886932373047 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 6.683923721313477 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 6.683159351348877 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 6.6840715408325195 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 6.6867146492004395 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 6.685583114624023 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 6.6854166984558105 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 6.682716369628906 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 6.680431365966797 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 6.677369117736816 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 6.679357051849365 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 6.684789657592773 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 6.683804035186768 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 6.686559677124023 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 6.687420845031738 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 6.677921772003174 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 6.678563594818115 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 6.672532081604004 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 6.6840500831604 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 6.684474945068359 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 6.6825690269470215 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 6.682358741760254 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 6.683840751647949 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 6.685976028442383 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 6.683504581451416 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 6.684739589691162 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 6.684548377990723 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 6.684298038482666 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 6.684053897857666 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 6.681194305419922 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 6.682559013366699 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 6.678596019744873 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 6.683594226837158 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 6.684631824493408 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 6.683034420013428 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 6.681471347808838 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 6.6816582679748535 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 6.679253101348877 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 6.677774906158447 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 6.685138702392578 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 6.684926986694336 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 6.687495231628418 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 6.68556547164917 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 6.684604167938232 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 6.68281888961792 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 6.6800537109375 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 6.685257434844971 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 6.68546724319458 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 6.684081554412842 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 6.682966709136963 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 6.679513931274414 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 6.679571628570557 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 6.6802897453308105 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 6.685147762298584 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 6.684807777404785 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 6.683252811431885 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 6.68406867980957 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 6.683474063873291 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 6.682219982147217 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 6.679166793823242 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 6.6847147941589355 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 6.684130668640137 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 6.680452346801758 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 6.68399715423584 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 6.680609703063965 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 6.678053379058838 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 6.679429531097412 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 6.684391975402832 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 6.683762550354004 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 6.682073593139648 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 6.682203769683838 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 6.680572032928467 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 6.681305885314941 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 6.680699348449707 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 6.684282302856445 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 6.684886455535889 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 6.684338092803955 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 6.684338092803955 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 6.677938461303711 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 6.685318470001221 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 6.676648139953613 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 6.684137344360352 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 6.685152530670166 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 6.684114933013916 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 6.68667459487915 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 6.680209636688232 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 6.678582668304443 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 6.682275772094727 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 6.6839776039123535 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 6.683745861053467 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 6.68242883682251 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 6.683191776275635 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 6.682838439941406 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 6.680160999298096 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 6.67639684677124 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 6.684627532958984 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 6.6853437423706055 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 6.683043956756592 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 6.684430122375488 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 6.680203437805176 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 6.681128978729248 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 6.681099891662598 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 6.684287071228027 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 6.684645652770996 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 6.684048652648926 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 6.683620929718018 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 6.681904315948486 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 6.681143283843994 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 6.6787872314453125 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 6.685009479522705 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 6.685187339782715 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 6.684188365936279 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 6.684294700622559 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 6.679564952850342 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 6.684985160827637 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 6.678619384765625 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 6.685090065002441 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 6.686267852783203 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 6.686182975769043 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 6.684610366821289 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 6.682829856872559 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 6.676028728485107 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 6.6793694496154785 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 6.685044765472412 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 6.68488883972168 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 6.6878509521484375 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 6.68579626083374 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 6.6803693771362305 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 6.682003021240234 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 6.676815986633301 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 6.684789180755615 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 6.684373378753662 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 6.684057235717773 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 6.684578895568848 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 6.680606365203857 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 6.682669639587402 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 6.680655002593994 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 6.685013294219971 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 6.684746265411377 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 6.684366703033447 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 6.68284797668457 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 6.679654121398926 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 6.681861877441406 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 6.678966522216797 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 6.6839213371276855 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 6.684873580932617 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 6.682742118835449 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 6.68410062789917 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 6.685760021209717 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 6.678065776824951 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 6.682178020477295 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 6.685023784637451 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 6.6845879554748535 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 6.682865619659424 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 6.683200359344482 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 6.681550025939941 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 6.681911468505859 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 6.681915760040283 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 6.685413837432861 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 6.685605525970459 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 6.683565139770508 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 6.6831231117248535 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 6.679602146148682 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 6.678132057189941 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 6.686398983001709 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 6.685206890106201 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 6.684651851654053 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 6.684645652770996 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 6.685779571533203 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 6.679839134216309 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 6.6814494132995605 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 6.678165435791016 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 6.684492588043213 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 6.6832194328308105 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 6.6829118728637695 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 6.6845502853393555 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 6.680050373077393 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 6.680342674255371 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 6.679011821746826 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 6.683608531951904 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 6.684850692749023 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 6.682596206665039 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 6.683945178985596 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 6.681829929351807 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 6.685525417327881 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 6.675976753234863 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 6.684167385101318 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 6.684317588806152 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 6.685938358306885 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 6.682830333709717 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 6.68102502822876 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 6.683472633361816 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 6.678884506225586 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 6.679993629455566 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 6.680627822875977 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 6.693792819976807 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 6.682142734527588 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 6.6756911277771 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 6.698426723480225 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 6.679279804229736 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 6.684568881988525 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 6.687106609344482 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 6.683122158050537 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 6.682763576507568 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 6.668097019195557 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 6.68671989440918 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 6.668574333190918 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 6.697816848754883 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 6.695910930633545 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 6.682331562042236 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 6.689478397369385 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 6.673758029937744 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 6.688999652862549 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 6.69936466217041 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 6.680384635925293 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 6.694886684417725 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 6.671898365020752 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 6.6777520179748535 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 6.676563739776611 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 6.68825626373291 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 6.681289196014404 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 6.697885990142822 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 6.689760208129883 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 6.680883884429932 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 6.681336402893066 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 6.6745710372924805 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 6.681718349456787 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 6.669277191162109 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 6.7014689445495605 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 6.760451316833496 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 6.691965579986572 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 6.685722827911377 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 6.657800197601318 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 6.695643901824951 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 6.7046051025390625 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 6.687724590301514 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 6.687488079071045 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 6.692960262298584 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 6.676198482513428 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 6.672645092010498 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 6.693437099456787 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 6.700255393981934 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 6.688964366912842 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 6.687915325164795 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 6.692327499389648 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 6.687088966369629 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 6.671365261077881 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 6.695592403411865 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 6.686121463775635 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 6.681429862976074 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 6.677724361419678 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 6.674583911895752 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 6.6793212890625 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 6.673661708831787 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 6.694416046142578 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 6.689381122589111 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 6.687681674957275 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 6.687443733215332 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 6.694998264312744 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 6.68858003616333 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 6.6292948722839355 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 6.633982181549072 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 6.636560916900635 + } + ] + }, + { + "bit": 4, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 5.491048812866211 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 5.4842753410339355 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 5.49097204208374 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 5.491086006164551 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 5.496713638305664 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 5.698660373687744 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 5.446321487426758 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 5.4853291511535645 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 5.478773593902588 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 5.461949348449707 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 5.531428813934326 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 5.512465476989746 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 5.468174457550049 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 5.462309837341309 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 5.478217601776123 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 5.477337837219238 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 5.445638656616211 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 5.487821578979492 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 5.505704879760742 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 5.543269634246826 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 5.54059362411499 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 5.657157897949219 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.50093936920166 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 5.500646114349365 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 5.50915002822876 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 5.48107385635376 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 5.478463649749756 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 5.482114315032959 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 5.4841508865356445 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 5.484899044036865 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 5.4844584465026855 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 5.483763694763184 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 5.482029438018799 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 5.483312129974365 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 5.483063697814941 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 5.484189033508301 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 5.483982563018799 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 5.483624458312988 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 5.483254909515381 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 5.483471393585205 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 5.4818830490112305 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 5.4837565422058105 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 5.484116554260254 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 5.4841814041137695 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 5.483165264129639 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 5.483365058898926 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 5.483349800109863 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 5.483718395233154 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 5.484413146972656 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 5.483682632446289 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 5.483679294586182 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 5.483345985412598 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 5.483222007751465 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 5.482985973358154 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 5.484109878540039 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 5.488707065582275 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 5.484297275543213 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 5.483621597290039 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 5.484927654266357 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 5.48301362991333 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 5.482507705688477 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 5.4822516441345215 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 5.483999729156494 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 5.483888149261475 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 5.4839324951171875 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 5.483307361602783 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 5.484186172485352 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 5.482393741607666 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 5.482386112213135 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 5.482715606689453 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 5.484157085418701 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 5.483870506286621 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 5.484794616699219 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 5.482843399047852 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 5.482260227203369 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 5.483536243438721 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 5.482004642486572 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 5.483644008636475 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 5.483488082885742 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 5.484604835510254 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 5.482433319091797 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 5.481986045837402 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 5.483819961547852 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 5.484415054321289 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 5.485423564910889 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 5.4844651222229 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 5.485383987426758 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 5.484141826629639 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 5.484429836273193 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 5.483326435089111 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 5.483429431915283 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 5.482405662536621 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 5.4817585945129395 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 5.484408855438232 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 5.485450744628906 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 5.484954357147217 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 5.484701633453369 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 5.484487056732178 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 5.482272624969482 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 5.481042861938477 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 5.484474182128906 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 5.484508037567139 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 5.485043525695801 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 5.484258651733398 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 5.485869407653809 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 5.48415470123291 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 5.48397970199585 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 5.485045433044434 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 5.485284805297852 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 5.485258102416992 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 5.482266902923584 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 5.481714248657227 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 5.482790946960449 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 5.483945846557617 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 5.4841108322143555 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 5.4817705154418945 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 5.483488082885742 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 5.481814384460449 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 5.48184871673584 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 5.483955383300781 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 5.483339786529541 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 5.479925632476807 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 5.477819919586182 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 5.484706878662109 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 5.483502388000488 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 5.482590198516846 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 5.483074188232422 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 5.484501838684082 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 5.483182430267334 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 5.486299514770508 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 5.481473445892334 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 5.483958721160889 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 5.481858253479004 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 5.484041690826416 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 5.484652996063232 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 5.481482982635498 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 5.480032444000244 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 5.482419967651367 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 5.482913017272949 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 5.477826118469238 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 5.484140396118164 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 5.484358310699463 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 5.481139183044434 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 5.484139919281006 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 5.483255386352539 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 5.481006145477295 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 5.4814019203186035 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 5.483081817626953 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 5.483273506164551 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 5.47818660736084 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 5.481656551361084 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 5.482310771942139 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 5.484941005706787 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 5.481160640716553 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 5.480694770812988 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 5.482532024383545 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 5.475281238555908 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 5.480165958404541 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 5.480164527893066 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 5.4845123291015625 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 5.485634803771973 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 5.481903553009033 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 5.482483863830566 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 5.486189842224121 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 5.479717254638672 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 5.480340957641602 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 5.48565149307251 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 5.484560966491699 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 5.4841718673706055 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 5.484395980834961 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 5.476803779602051 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 5.481902599334717 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 5.482583522796631 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 5.481110572814941 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 5.482668399810791 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 5.483255386352539 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 5.4841628074646 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 5.485320091247559 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 5.4823832511901855 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 5.476963043212891 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 5.48178243637085 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 5.48032808303833 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 5.483951091766357 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 5.483991622924805 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 5.484777450561523 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 5.482625484466553 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 5.4800190925598145 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 5.480899810791016 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 5.480264663696289 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 5.48222541809082 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 5.482793807983398 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 5.481036186218262 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 5.482365131378174 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 5.48222541809082 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 5.480437755584717 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 5.480319976806641 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 5.482337474822998 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 5.482510566711426 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 5.480642795562744 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 5.48211145401001 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 5.4828410148620605 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 5.481076717376709 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 5.48211145401001 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 5.482666969299316 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 5.483067989349365 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 5.484302520751953 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 5.479533672332764 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 5.479342937469482 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 5.4801177978515625 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 5.477931022644043 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 5.481186389923096 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 5.483256816864014 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 5.482088088989258 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 5.484460830688477 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 5.48093318939209 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 5.4802446365356445 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 5.48310661315918 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 5.481921195983887 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 5.482142925262451 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 5.477852821350098 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 5.480632781982422 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 5.482858657836914 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 5.485307216644287 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 5.481617450714111 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 5.483317852020264 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 5.484005928039551 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 5.488015174865723 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 5.484833717346191 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 5.483491897583008 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 5.483315944671631 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 5.4842071533203125 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 5.486152172088623 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 5.483764171600342 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 5.485386848449707 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 5.482976913452148 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 5.483450412750244 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 5.4847917556762695 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 5.483765125274658 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 5.486198425292969 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 5.487614631652832 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 5.484752655029297 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 5.485658645629883 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 5.474471569061279 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 5.47928524017334 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 5.481900215148926 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 5.484659671783447 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 5.483302593231201 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 5.482945919036865 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 5.483621597290039 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 5.483875274658203 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 5.483484268188477 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 5.482796669006348 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 5.480757713317871 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 5.4842095375061035 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 5.47774076461792 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 5.480967044830322 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 5.481410026550293 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 5.482000827789307 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 5.485254764556885 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 5.483782768249512 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 5.482494831085205 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 5.4813079833984375 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 5.481373310089111 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 5.48407506942749 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 5.481784343719482 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 5.482514381408691 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 5.484968662261963 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 5.48428201675415 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 5.483546733856201 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 5.482054233551025 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 5.482244491577148 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 5.480875492095947 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 5.482245922088623 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 5.482656478881836 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 5.48404598236084 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 5.482293128967285 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 5.4820122718811035 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 5.482724666595459 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 5.481868267059326 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 5.481274127960205 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 5.483382225036621 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 5.48350191116333 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 5.482746124267578 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 5.481208324432373 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 5.481457233428955 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 5.483509540557861 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 5.480296611785889 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 5.483604431152344 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 5.484229564666748 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 5.484065055847168 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 5.484215259552002 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 5.482593059539795 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 5.482260704040527 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 5.482244491577148 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 5.483791828155518 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 5.4839324951171875 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 5.483566761016846 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 5.4830403327941895 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 5.482528209686279 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 5.480663776397705 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 5.480654716491699 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 5.485231876373291 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 5.484622955322266 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 5.484303951263428 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 5.483845233917236 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 5.481008052825928 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 5.484336853027344 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 5.479629039764404 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 5.484025478363037 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 5.48361349105835 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 5.484910011291504 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 5.482789516448975 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 5.484652519226074 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 5.482489109039307 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 5.48284387588501 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 5.483419895172119 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 5.4838714599609375 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 5.483336925506592 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 5.483218669891357 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 5.482821941375732 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 5.484391212463379 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 5.482620716094971 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 5.484258651733398 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 5.484060764312744 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 5.483557224273682 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 5.483068466186523 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 5.4815568923950195 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 5.483304023742676 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 5.481179237365723 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 5.48461389541626 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 5.484412670135498 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 5.4845709800720215 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 5.482922077178955 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 5.482263565063477 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 5.481543064117432 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 5.482570648193359 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 5.48454475402832 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 5.4842400550842285 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 5.484238624572754 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 5.484817981719971 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 5.48208475112915 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 5.479439735412598 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 5.481511116027832 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 5.485621929168701 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 5.4849324226379395 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 5.485631942749023 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 5.483923435211182 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 5.482576370239258 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 5.4822587966918945 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 5.483205795288086 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 5.484417915344238 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 5.4846110343933105 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 5.484094142913818 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 5.484111309051514 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 5.48225736618042 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 5.481229305267334 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 5.482415199279785 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 5.484196662902832 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 5.4842658042907715 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 5.484982013702393 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 5.485202312469482 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 5.483036041259766 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 5.479953765869141 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 5.480665683746338 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 5.483888149261475 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 5.485215663909912 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 5.483268737792969 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 5.483198165893555 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 5.483202934265137 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 5.480615139007568 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 5.483422756195068 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 5.484348297119141 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 5.484225749969482 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 5.483247756958008 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 5.4846882820129395 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 5.4813008308410645 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 5.481154918670654 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 5.479893684387207 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 5.48397159576416 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 5.483555793762207 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 5.48445463180542 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 5.484388828277588 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 5.4847235679626465 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 5.4808349609375 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 5.484816551208496 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 5.483953952789307 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 5.484111785888672 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 5.483926296234131 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 5.484100341796875 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 5.481991767883301 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 5.481326103210449 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 5.481746673583984 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 5.484522342681885 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 5.4848151206970215 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 5.484570026397705 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 5.484474182128906 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 5.483279228210449 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 5.484001159667969 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 5.481032371520996 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 5.484439849853516 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 5.48417329788208 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 5.4845170974731445 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 5.483786106109619 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 5.48237419128418 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 5.482278823852539 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 5.483523368835449 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 5.483987808227539 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 5.484030723571777 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 5.4845123291015625 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 5.484447956085205 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 5.480415344238281 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 5.480979919433594 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 5.481715679168701 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 5.484102725982666 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 5.484120845794678 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 5.483657360076904 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 5.485097408294678 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 5.4823503494262695 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 5.481369495391846 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 5.48250675201416 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 5.484188556671143 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 5.484323501586914 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 5.484834671020508 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 5.484297275543213 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 5.4828267097473145 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 5.480031490325928 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 5.482185363769531 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 5.483933925628662 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 5.483870506286621 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 5.484335422515869 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 5.483950138092041 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 5.482229709625244 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 5.482601165771484 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 5.48012638092041 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 5.4845733642578125 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 5.484377384185791 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 5.484239101409912 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 5.484302520751953 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 5.482438087463379 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 5.482633113861084 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 5.482842445373535 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 5.484548568725586 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 5.4842023849487305 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 5.484035015106201 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 5.4831719398498535 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 5.482266902923584 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 5.48419189453125 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 5.481543064117432 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 5.4840474128723145 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 5.483951091766357 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 5.48307991027832 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 5.483497619628906 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 5.4830451011657715 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 5.480907440185547 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 5.482631206512451 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 5.48480749130249 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 5.484661102294922 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 5.48494815826416 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 5.484565734863281 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 5.483033180236816 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 5.482677459716797 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 5.482386589050293 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 5.484531402587891 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 5.483938217163086 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 5.483336925506592 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 5.483425617218018 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 5.4829630851745605 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 5.480415344238281 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 5.481795787811279 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 5.483806133270264 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 5.483677864074707 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 5.485661506652832 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 5.483177661895752 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 5.4810872077941895 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 5.4813666343688965 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 5.483859539031982 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 5.483220100402832 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 5.483908653259277 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 5.4824042320251465 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 5.484128952026367 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 5.480574131011963 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 5.480078220367432 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 5.482105731964111 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 5.485583305358887 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 5.484416484832764 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 5.482979774475098 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 5.482284069061279 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 5.481841564178467 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 5.4798665046691895 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 5.481605529785156 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 5.481956481933594 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 5.482487678527832 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 5.481381893157959 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 5.4819536209106445 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 5.479576587677002 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 5.484964370727539 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 5.476682186126709 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 5.479762077331543 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 5.4851460456848145 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 5.47737455368042 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 5.483025550842285 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 5.472017288208008 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 5.490241527557373 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 5.481183052062988 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 5.479578018188477 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 5.482166290283203 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 5.483067035675049 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 5.481406211853027 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 5.47705602645874 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 5.485235691070557 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 5.481860160827637 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 5.478679180145264 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 5.488534450531006 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 5.475853443145752 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 5.4787068367004395 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 5.475531101226807 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 5.483718395233154 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 5.479496002197266 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 5.483469486236572 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 5.486536502838135 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 5.486505031585693 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 5.482288360595703 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 5.470633029937744 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 5.48151969909668 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 5.479369163513184 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 5.501288414001465 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 5.50504732131958 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 5.487637042999268 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 5.4841718673706055 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 5.475773811340332 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 5.491720199584961 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 5.483447551727295 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 5.488460063934326 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 5.492976188659668 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 5.480998992919922 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 5.481006145477295 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 5.4829277992248535 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 5.487581253051758 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 5.4855122566223145 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 5.4871697425842285 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 5.491366386413574 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 5.482874870300293 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 5.48442268371582 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 5.482931137084961 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 5.489919185638428 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 5.481916904449463 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 5.484429836273193 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 5.4823126792907715 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 5.480544090270996 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 5.480647087097168 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 5.4721879959106445 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 5.489014148712158 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 5.480921268463135 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 5.487478733062744 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 5.485933303833008 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 5.484277725219727 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 5.486446380615234 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 5.468066215515137 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 5.466848373413086 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 5.453749656677246 + } + ] + }, + { + "bit": 5, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.6950459480285645 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.697357654571533 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.693120956420898 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.718608379364014 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.696907997131348 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.694621562957764 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.6976470947265625 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.694665908813477 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.700019836425781 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.6981306076049805 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.7572784423828125 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.699825286865234 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.691462993621826 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.691714763641357 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.701437950134277 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.696896076202393 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.6975579261779785 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.693696975708008 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.691248893737793 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.693277359008789 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.690928936004639 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.6908087730407715 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 5.10003662109375 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.693862438201904 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.6933159828186035 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.693357467651367 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.701474666595459 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.693053722381592 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.694252967834473 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.694220542907715 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.69374418258667 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.693812370300293 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.69342041015625 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.692935943603516 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.693296432495117 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.6940436363220215 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.6942644119262695 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.6935296058654785 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.694113254547119 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.693193435668945 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.694161415100098 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.693581581115723 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.693989276885986 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.69443416595459 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.693597316741943 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.6943535804748535 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.69392728805542 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.693822860717773 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.694177150726318 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.6944169998168945 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.694272041320801 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.694105625152588 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.694070339202881 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.693889617919922 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.693905830383301 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.69464111328125 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.693770885467529 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.694326877593994 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.6944403648376465 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.694097518920898 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.694126129150391 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.694523811340332 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.69392204284668 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.694459438323975 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.694027423858643 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.694544315338135 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.693710803985596 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.694108963012695 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.693521499633789 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.693655967712402 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.694499492645264 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.694551467895508 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.69429874420166 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.694272041320801 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.6941609382629395 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.6937456130981445 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.694643020629883 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.693905830383301 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.694242000579834 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.693676948547363 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.69425106048584 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.693480968475342 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.693349838256836 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.69392204284668 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.694268703460693 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.694087505340576 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.695105075836182 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.693858623504639 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.693599700927734 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.694051265716553 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.694694519042969 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.693413257598877 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.693282604217529 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.693933963775635 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.694591999053955 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.694827079772949 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.694291591644287 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.693924903869629 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.693384647369385 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.69379997253418 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.694445610046387 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.693238258361816 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.694510459899902 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.693904876708984 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.694943904876709 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.694278717041016 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.693758010864258 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.692921161651611 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.693113803863525 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.693843841552734 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.6938557624816895 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.693802356719971 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.69395112991333 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.694002151489258 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.693826675415039 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.694161891937256 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.693778038024902 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.694186687469482 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.694252967834473 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.693666934967041 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.693885803222656 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.693249702453613 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.693862438201904 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.693528175354004 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.693849086761475 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.693397521972656 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.693572521209717 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.695518970489502 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.691850662231445 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.692823886871338 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.694266319274902 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.693075656890869 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.693049907684326 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.694070339202881 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.694765567779541 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.692404270172119 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.694075107574463 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.694283485412598 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.693276405334473 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.693781852722168 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.694286346435547 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.694064140319824 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.6936235427856445 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.693820953369141 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.694225311279297 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.694710731506348 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.693596363067627 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.69403600692749 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.693843841552734 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.693784713745117 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.693003177642822 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.693543434143066 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.696134567260742 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.691961288452148 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.6926774978637695 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.692807674407959 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.694579601287842 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.692824840545654 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.692941665649414 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.693961143493652 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.693016529083252 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.693451881408691 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.693527698516846 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.689280986785889 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.691894054412842 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.692905902862549 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.693772315979004 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.694200038909912 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.693819999694824 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.694723129272461 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.693856716156006 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.69367790222168 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.691479682922363 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.694371700286865 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.693488597869873 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.693325519561768 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.693507671356201 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.693046569824219 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.692810535430908 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.692721843719482 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.692030906677246 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.6930413246154785 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.694329261779785 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.693816184997559 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.693922996520996 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.693357467651367 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.692256450653076 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.69416618347168 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.69372034072876 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.6926589012146 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.69350004196167 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.692464828491211 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.691555023193359 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.693936347961426 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.692823886871338 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.692983150482178 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.693817615509033 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.6933064460754395 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.695940971374512 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.692958354949951 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.694797992706299 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.693178653717041 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.6934661865234375 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.694035053253174 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.693559169769287 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.694072723388672 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.69416618347168 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.693034648895264 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.694225788116455 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.6931562423706055 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.693908214569092 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.694189071655273 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.6938581466674805 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.69303035736084 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.693392753601074 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.692520618438721 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.691756725311279 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.6936492919921875 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.692764759063721 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.694369792938232 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.693791389465332 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.69278621673584 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.693019866943359 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.6935296058654785 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.693694591522217 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.693426132202148 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.692705154418945 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.694265842437744 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.693079471588135 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.692866802215576 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.692718029022217 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.6944379806518555 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.694157600402832 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.693124771118164 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.693892002105713 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.693850040435791 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.694591999053955 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.692902565002441 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.695809841156006 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.6944708824157715 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.697202205657959 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.693391799926758 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.6921000480651855 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.691823959350586 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.693633079528809 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.694360733032227 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.693922996520996 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.692254543304443 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.692983150482178 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.692898750305176 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.694469451904297 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.692532539367676 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.69371223449707 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.694446086883545 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.693924903869629 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.692906379699707 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.695141792297363 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.693887233734131 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.692132949829102 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.693727493286133 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.693757057189941 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.694016933441162 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.695187568664551 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.693758964538574 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.6927313804626465 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.693938732147217 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.69420051574707 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.69426155090332 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.693023204803467 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.693483352661133 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.693699836730957 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.692375659942627 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.691773891448975 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.693857669830322 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.694210529327393 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.692674160003662 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.69278621673584 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.692849636077881 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.692943096160889 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.692315101623535 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.694085121154785 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.693594932556152 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.693902969360352 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.694116115570068 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.693355560302734 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.693727493286133 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.692876815795898 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.693869113922119 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.6942901611328125 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.693413734436035 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.693272590637207 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.693270206451416 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.693444728851318 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.690572261810303 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.694031238555908 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.693917274475098 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.694286346435547 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.6936564445495605 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.693444728851318 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.692668437957764 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.693645477294922 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.694705009460449 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.694762706756592 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.694093227386475 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.694343566894531 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.692534446716309 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.69296407699585 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.694203853607178 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.694417476654053 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.69388484954834 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.69366979598999 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.693820953369141 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.692630767822266 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.692280292510986 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.6923298835754395 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.6939802169799805 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.694275379180908 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.69464635848999 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.693735122680664 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.692670822143555 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.6920318603515625 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.6936354637146 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.694179058074951 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.6940741539001465 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.69370698928833 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.693601608276367 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.692877769470215 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.692628383636475 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.6925506591796875 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.694497108459473 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.694310188293457 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.694423675537109 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.693795680999756 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.693377494812012 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.692544460296631 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.692770481109619 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.694126605987549 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.694179058074951 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.693390369415283 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.6937336921691895 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.693540096282959 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.693033695220947 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.6925129890441895 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.694443225860596 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.694578647613525 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.6936750411987305 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.693276405334473 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.693223476409912 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.692523002624512 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.691730499267578 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.694125652313232 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.694220066070557 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.694242477416992 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.693361759185791 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.693206787109375 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.692859172821045 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.692905902862549 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.693881988525391 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.694000244140625 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.693855285644531 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.693477630615234 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.692894458770752 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.692242622375488 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.691989898681641 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.694122314453125 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.6944580078125 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.6937031745910645 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.692853927612305 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.692831993103027 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.693750381469727 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.690831661224365 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.694134712219238 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.694450855255127 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.6941938400268555 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.693974494934082 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.692902088165283 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.6938982009887695 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.693929672241211 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.694294452667236 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.694085121154785 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.69430685043335 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.69404935836792 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.692618370056152 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.693262577056885 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.69174337387085 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.694088935852051 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.6942219734191895 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.694733619689941 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.693761825561523 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.692043304443359 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.69288969039917 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.692473411560059 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.694247245788574 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.694177150726318 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.694742202758789 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.693989276885986 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.692328929901123 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.692214488983154 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.691222190856934 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.694174766540527 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.694218158721924 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.693868160247803 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.694102764129639 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.692548751831055 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.693454265594482 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.691677570343018 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.694138050079346 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.694206237792969 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.694151401519775 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.694025993347168 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.6942219734191895 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.693063735961914 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.692680358886719 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.694094657897949 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.6941423416137695 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.694282531738281 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.694088935852051 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.6919074058532715 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.693532466888428 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.692204475402832 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.694089412689209 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.693866729736328 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.693872928619385 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.693817138671875 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.693963050842285 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.693732261657715 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.69285774230957 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.694281578063965 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.693997859954834 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.693939208984375 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.693382740020752 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.692194938659668 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.693814754486084 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.693779468536377 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.694007396697998 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.694092750549316 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.694085121154785 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.694199085235596 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.692307472229004 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.690942764282227 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.692336082458496 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.694095134735107 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.694357395172119 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.693794250488281 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.6938557624816895 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.6927266120910645 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.691946029663086 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.693321704864502 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.694004058837891 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.6939377784729 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.693957328796387 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.693865776062012 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.6954240798950195 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.692270278930664 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.692460060119629 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.694088935852051 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.694479465484619 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.693957328796387 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.6943254470825195 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.6928300857543945 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.6928935050964355 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.69268274307251 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.694140911102295 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.694055080413818 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.693564414978027 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.693713188171387 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.693259239196777 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.692088603973389 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.692644119262695 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.694316387176514 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.69359016418457 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.694884300231934 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.694033622741699 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.693477630615234 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.692286968231201 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.692376613616943 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.693317890167236 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.693820953369141 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.693594932556152 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.693477630615234 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.69237756729126 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.695194244384766 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.692022800445557 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.695055961608887 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.694281101226807 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.693865776062012 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.694865703582764 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.693160057067871 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.694082736968994 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.692718029022217 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.693783283233643 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.6935648918151855 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.694904327392578 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.693104267120361 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.692300796508789 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.6961236000061035 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.6911773681640625 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.691438674926758 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.6940412521362305 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.689965724945068 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.693143844604492 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.689957618713379 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.696422576904297 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.694635391235352 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.6917877197265625 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.69508695602417 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.69138765335083 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.693058967590332 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.689777374267578 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.692800045013428 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.694669246673584 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.691778182983398 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.696122169494629 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.694719314575195 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.692248821258545 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.690982341766357 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.693795680999756 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.693533420562744 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.695221900939941 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.695613861083984 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.693164825439453 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.694181442260742 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.691023826599121 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.692895889282227 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.685635089874268 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.708106517791748 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.702073097229004 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.683043003082275 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.694538116455078 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.6861677169799805 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.6973772048950195 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.694140911102295 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.693470001220703 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.696556091308594 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.693560600280762 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.690793514251709 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.693233489990234 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.694956302642822 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.700142860412598 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.694016933441162 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.696432113647461 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.691985130310059 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.693274974822998 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.690014362335205 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.695043563842773 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.693127155303955 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.693231105804443 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.693393707275391 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.694055080413818 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.692659854888916 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.690902233123779 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.697964191436768 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.692732810974121 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.694761276245117 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.695127964019775 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.695037364959717 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.6943039894104 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.686300277709961 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.689430236816406 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.6741790771484375 + } + ] + }, + { + "bit": 6, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.386018753051758 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.386787414550781 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.393523693084717 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.391900539398193 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.387314319610596 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.384573459625244 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.384546279907227 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.38769006729126 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.386148452758789 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.384445667266846 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.386998653411865 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.38646936416626 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.384117126464844 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.3847222328186035 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.387123107910156 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.385988235473633 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.392747402191162 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.387153625488281 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.385952472686768 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.384300231933594 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.38510799407959 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.384711742401123 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.38470983505249 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.389882564544678 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.384873390197754 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.386602401733398 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.3853020668029785 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.385904788970947 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.386642932891846 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.386553764343262 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.386916160583496 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.386491775512695 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.386434078216553 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.386250972747803 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.386212348937988 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.386518955230713 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.386584281921387 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.386545658111572 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.3865203857421875 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.3863525390625 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.386576175689697 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.38591194152832 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.386922359466553 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.3866472244262695 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.386638641357422 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.386694431304932 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.386180877685547 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.386484622955322 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.386105537414551 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.386584281921387 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.386565685272217 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.386279106140137 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.386436939239502 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.386602401733398 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.386318683624268 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.386586666107178 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.386653900146484 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.3866424560546875 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.386623382568359 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.386815547943115 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.3862714767456055 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.386564254760742 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.386592864990234 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.386542797088623 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.386526584625244 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.386408805847168 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.386662483215332 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.386409759521484 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.386692047119141 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.386419296264648 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.386760234832764 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.386744976043701 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.386206150054932 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.386210918426514 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.386845588684082 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.386702537536621 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.3862624168396 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.386586666107178 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.386497974395752 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.386555194854736 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.386553764343262 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.386204242706299 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.386512279510498 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.386791229248047 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.387016296386719 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.387143135070801 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.3867340087890625 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.387125015258789 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.386528015136719 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.385986804962158 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.386361122131348 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.38608980178833 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.386192321777344 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.38627290725708 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.38606071472168 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.386458396911621 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.386029243469238 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.3867058753967285 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.386849880218506 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.386476039886475 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.386729717254639 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.386266708374023 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.38657808303833 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.386124610900879 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.386798858642578 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.386472225189209 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.386734962463379 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.386890888214111 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.3869218826293945 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.386932849884033 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.387415409088135 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.386001110076904 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.386721134185791 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.386794090270996 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.387181282043457 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.386674404144287 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.386895179748535 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.3859686851501465 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.386189937591553 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.386538028717041 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.386260986328125 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.386013984680176 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.3867506980896 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.387078285217285 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.386868000030518 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.386619567871094 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.386160850524902 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.387170314788818 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.386570930480957 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.385990142822266 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.386602401733398 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.386729717254639 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.386166572570801 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.386284351348877 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.3869829177856445 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.386394023895264 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.386148452758789 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.385667324066162 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.386674404144287 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.385809421539307 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.386287689208984 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.386631488800049 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.387197494506836 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.385777473449707 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.386040687561035 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.386999607086182 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.3861613273620605 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.386942386627197 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.3864312171936035 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.384400367736816 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.386393070220947 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.385477542877197 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.38638162612915 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.386995792388916 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.385617733001709 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.385700702667236 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.385202407836914 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.385960102081299 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.3859734535217285 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.3871917724609375 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.385886192321777 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.386270046234131 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.386702537536621 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.385743141174316 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.38636589050293 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.387228012084961 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.386196613311768 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.3868303298950195 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.3866472244262695 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.386037349700928 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.3837761878967285 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.385678291320801 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.385478973388672 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.386746406555176 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.386963844299316 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.386270523071289 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.385815143585205 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.386695861816406 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.386533260345459 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.385946750640869 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.385812759399414 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.385952472686768 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.386794090270996 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.38653564453125 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.387338161468506 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.386201858520508 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.3866658210754395 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.385406970977783 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.385563850402832 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.386058807373047 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.386166095733643 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.386995315551758 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.385736465454102 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.385732173919678 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.386761665344238 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.385643005371094 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.386254787445068 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.3862738609313965 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.385432720184326 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.3853607177734375 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.385361671447754 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.385927677154541 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.385037899017334 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.386323928833008 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.386295795440674 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.386310577392578 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.386392593383789 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.386312484741211 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.38559103012085 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.385372638702393 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.386001110076904 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.385993480682373 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.385943412780762 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.3855671882629395 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.3861308097839355 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.385791778564453 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.385627269744873 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.3863396644592285 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.386317729949951 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.385495662689209 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.386044502258301 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.38608455657959 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.3861188888549805 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.386097431182861 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.38595724105835 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.386457443237305 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.386769771575928 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.385476112365723 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.386566638946533 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.386765956878662 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.3870439529418945 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.38671350479126 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.386350154876709 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.386321067810059 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.386453151702881 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.386224746704102 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.387203693389893 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.387226581573486 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.386415481567383 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.387091636657715 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.385586738586426 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.386433124542236 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.385353088378906 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.386188507080078 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.385629177093506 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.387138843536377 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.38635778427124 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.386251926422119 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.386507034301758 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.386173248291016 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.386104583740234 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.386490345001221 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.385998725891113 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.386250972747803 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.386701583862305 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.386112689971924 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.385523796081543 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.385732650756836 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.387328147888184 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.38632345199585 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.386685848236084 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.385738372802734 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.385128021240234 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.386673450469971 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.38605260848999 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.385652542114258 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.386653423309326 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.386666774749756 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.386772632598877 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.386357307434082 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.386080741882324 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.387544631958008 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.386466026306152 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.386153697967529 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.385994911193848 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.385922431945801 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.385820388793945 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.386659622192383 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.386526107788086 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.386073589324951 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.386356353759766 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.386455535888672 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.38673210144043 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.385837078094482 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.386221408843994 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.385635852813721 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.3860249519348145 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.386214733123779 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.38668155670166 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.3866658210754395 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.38629150390625 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.385799884796143 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.386349201202393 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.386706352233887 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.386667251586914 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.386491775512695 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.386605262756348 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.385649681091309 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.385351657867432 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.385812759399414 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.385519027709961 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.386756896972656 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.386955261230469 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.386726379394531 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.38601541519165 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.386141300201416 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.385733604431152 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.385254383087158 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.3865132331848145 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.386458873748779 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.386957168579102 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.387536525726318 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.38543176651001 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.385898113250732 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.384632110595703 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.386669158935547 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.386641979217529 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.38651180267334 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.386541843414307 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.386117935180664 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.3854780197143555 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.3848185539245605 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.3866963386535645 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.386659622192383 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.386286735534668 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.386544227600098 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.386148452758789 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.386259078979492 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.386896133422852 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.386569023132324 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.386746406555176 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.3863525390625 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.386343955993652 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.3857951164245605 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.386235237121582 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.386305332183838 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.386672019958496 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.386387825012207 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.386780261993408 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.385910987854004 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.385950088500977 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.385791778564453 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.386951446533203 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.386939525604248 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.387226581573486 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.386783123016357 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.386895179748535 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.385720729827881 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.385815620422363 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.386033058166504 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.386625289916992 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.386501789093018 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.386206150054932 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.386415481567383 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.385951519012451 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.38733434677124 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.385098934173584 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.386773586273193 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.386627197265625 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.386297702789307 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.38673210144043 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.385244369506836 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.385567665100098 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.386250019073486 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.386122703552246 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.386682510375977 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.387130260467529 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.386777877807617 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.386778831481934 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.384861469268799 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.386890411376953 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.386632442474365 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.386629104614258 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.386311054229736 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.386528015136719 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.3848419189453125 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.386015892028809 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.385562896728516 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.386693000793457 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.386563301086426 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.386036396026611 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.386626243591309 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.386178970336914 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.386837482452393 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.386469841003418 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.386641025543213 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.386390686035156 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.3865556716918945 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.386036396026611 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.386135578155518 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.386463642120361 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.385735034942627 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.386751174926758 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.38657808303833 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.386427402496338 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.386655330657959 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.385720729827881 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.385869026184082 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.386225700378418 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.3867716789245605 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.3867926597595215 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.386397838592529 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.386377334594727 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.385155200958252 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.386120319366455 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.386550426483154 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.386707782745361 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.386622428894043 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.386449813842773 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.386474609375 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.3851823806762695 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.385152816772461 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.385694980621338 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.386510848999023 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.386475086212158 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.387002468109131 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.386353969573975 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.386709690093994 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.386241436004639 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.385737419128418 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.3867034912109375 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.386584281921387 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.386812210083008 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.38625955581665 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.385351657867432 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.386254787445068 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.385240077972412 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.386593818664551 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.386470794677734 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.386858940124512 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.386590480804443 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.385643005371094 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.386244297027588 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.386201858520508 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.3865556716918945 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.386547565460205 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.386448383331299 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.386656761169434 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.385328769683838 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.387274265289307 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.385898113250732 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.386891841888428 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.38677453994751 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.386399269104004 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.386430740356445 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.386293888092041 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.38490629196167 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.384729385375977 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.386594295501709 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.386457920074463 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.3868818283081055 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.386119842529297 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.385739803314209 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.3854193687438965 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.385763645172119 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.386789798736572 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.386839389801025 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.386614799499512 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.386803150177002 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.386764049530029 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.385796546936035 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.384790420532227 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.386663436889648 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.386623382568359 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.386650562286377 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.386253833770752 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.384908199310303 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.386150360107422 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.384548664093018 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.386617183685303 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.386574745178223 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.386519432067871 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.386904239654541 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.385549545288086 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.385006427764893 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.385777950286865 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.386326789855957 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.386139869689941 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.385951042175293 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.3855133056640625 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.385448932647705 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.38654899597168 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.385777473449707 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.38715124130249 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.3866682052612305 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.386585712432861 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.385768413543701 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.385386943817139 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.386277198791504 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.38556432723999 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.386552333831787 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.386590480804443 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.386489391326904 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.385226249694824 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.385464668273926 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.385989189147949 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.385582447052002 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.386438846588135 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.3867926597595215 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.384525775909424 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.386172294616699 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.383259296417236 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.389220714569092 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.384312152862549 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.386518478393555 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.385982990264893 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.384561061859131 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.385441303253174 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.384668350219727 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.3874359130859375 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.385244846343994 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.385326385498047 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.387041091918945 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.384199142456055 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.384167671203613 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.384570598602295 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.388487339019775 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.3871941566467285 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.3859148025512695 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.387250900268555 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.387642860412598 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.3851237297058105 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.382247447967529 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.385833263397217 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.386096000671387 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.390124797821045 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.392767429351807 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.385699272155762 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.386507034301758 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.3823018074035645 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.390312671661377 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.3874053955078125 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.387603282928467 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.387567520141602 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.386765480041504 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.385300159454346 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.385486125946045 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.38861083984375 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.388477325439453 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.386777877807617 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.3866868019104 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.385657787322998 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.386419296264648 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.384891510009766 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.387751579284668 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.388969898223877 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.3863420486450195 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.385097503662109 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.384944915771484 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.385646343231201 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.3835883140563965 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.389415264129639 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.387238025665283 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.387282371520996 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.387324810028076 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.386325836181641 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.387622833251953 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.382380962371826 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.38208532333374 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.377748012542725 + } + ] + }, + { + "bit": 7, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.223017692565918 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.2228779792785645 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.222786903381348 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.223099708557129 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.222897529602051 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.223183631896973 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.2230544090271 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.222636699676514 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.2226738929748535 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.223374843597412 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.222567558288574 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.222297191619873 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.222675800323486 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.22278356552124 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.222568511962891 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.222736835479736 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.223007678985596 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.222403049468994 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.222517013549805 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.222318649291992 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.222591400146484 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.222513198852539 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.222785472869873 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.222619533538818 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.222365379333496 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.222453594207764 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.221846103668213 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.222787857055664 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.222647666931152 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.222661972045898 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.222359657287598 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.223020076751709 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.2223639488220215 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.222133159637451 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.2222137451171875 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.222965240478516 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.222656726837158 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.223055362701416 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.2230939865112305 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.2228264808654785 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.223343372344971 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.222544193267822 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.222512245178223 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.222944736480713 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.222424507141113 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.222529888153076 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.222731113433838 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.222884654998779 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.222720146179199 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.222783088684082 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.222607612609863 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.222846984863281 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.2224907875061035 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.223033428192139 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.22252082824707 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.222708702087402 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.222720623016357 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.222722053527832 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.223074436187744 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.222908020019531 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.223030090332031 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.2221879959106445 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.222660541534424 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.2230119705200195 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.222644805908203 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.222241401672363 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.222583293914795 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.222867488861084 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.222834587097168 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.222350597381592 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.223053455352783 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.222496032714844 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.2229766845703125 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.2227325439453125 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.222720146179199 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.222753524780273 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.222933769226074 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.222692966461182 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.2227253913879395 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.222874164581299 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.222537994384766 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.223180294036865 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.222841739654541 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.222641468048096 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.22287130355835 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.22320556640625 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.2229766845703125 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.222805023193359 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.222706317901611 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.222919464111328 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.222865104675293 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.222378730773926 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.222443580627441 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.2227935791015625 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.222894668579102 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.2228240966796875 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.223153114318848 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.22286319732666 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.223036289215088 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.223038196563721 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.223287105560303 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.222360610961914 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.223339557647705 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.22256326675415 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.222851753234863 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.222991943359375 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.222569465637207 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.222755432128906 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.22292947769165 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.222821235656738 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.2228312492370605 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.222583293914795 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.2228474617004395 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.222789764404297 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.22260856628418 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.223038196563721 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.22271728515625 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.22340202331543 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.222995281219482 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.222987174987793 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.22262716293335 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.222593307495117 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.222351551055908 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.223243713378906 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.223020076751709 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.222938537597656 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.2230072021484375 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.222564697265625 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.222546100616455 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.222763538360596 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.2225341796875 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.223453044891357 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.222368240356445 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.222704887390137 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.222742557525635 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.222170829772949 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.222532272338867 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.222402572631836 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.222865581512451 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.222126483917236 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.222904205322266 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.222659587860107 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.223212242126465 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.223181247711182 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.223109722137451 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.223028182983398 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.22312593460083 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.222800254821777 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.222599506378174 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.2226386070251465 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.222189903259277 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.222755432128906 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.222805023193359 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.222926616668701 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.2223381996154785 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.22238302230835 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.222429275512695 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.221649169921875 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.2221574783325195 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.223074436187744 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.22299861907959 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.222578048706055 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.223039150238037 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.2210493087768555 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.221377372741699 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.221827507019043 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.22407865524292 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.223130226135254 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.222641468048096 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.222982883453369 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.223504543304443 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.2227888107299805 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.22174596786499 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.222367763519287 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.222426414489746 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.222496032714844 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.222997188568115 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.2227959632873535 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.222498893737793 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.221407890319824 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.222842693328857 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.222537517547607 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.222704887390137 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.222606182098389 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.2229132652282715 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.2228779792785645 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.222175598144531 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.223415851593018 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.222867965698242 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.2224907875061035 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.222379684448242 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.222350120544434 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.2221784591674805 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.22296667098999 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.222066879272461 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.223068714141846 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.222879409790039 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.222485542297363 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.223395824432373 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.222383975982666 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.223683834075928 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.222428321838379 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.222909450531006 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.222349166870117 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.222988128662109 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.22289514541626 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.222774505615234 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.222023963928223 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.222524166107178 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.222554683685303 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.222293853759766 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.222825527191162 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.22268009185791 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.223373889923096 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.22251033782959 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.222148418426514 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.2224531173706055 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.222480297088623 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.2221455574035645 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.223851680755615 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.222744464874268 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.222501754760742 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.222235202789307 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.222546100616455 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.222329616546631 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.222471237182617 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.222329616546631 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.223410129547119 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.222499847412109 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.222524166107178 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.221695423126221 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.22292947769165 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.222908973693848 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.222879409790039 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.222078800201416 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.222936153411865 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.222412109375 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.2230963706970215 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.223256587982178 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.2227935791015625 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.222949504852295 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.222472667694092 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.221989631652832 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.22230339050293 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.221443176269531 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.22274923324585 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.222524642944336 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.2224650382995605 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.222351551055908 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.222391605377197 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.223552227020264 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.22172212600708 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.222336769104004 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.223183631896973 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.221700191497803 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.222036361694336 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.223751068115234 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.222583293914795 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.222403049468994 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.222421169281006 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.222629070281982 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.222561836242676 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.222890853881836 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.222473621368408 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.223100662231445 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.2236199378967285 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.222589015960693 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.222567558288574 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.222589015960693 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.221694469451904 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.222429275512695 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.222047805786133 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.2206315994262695 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.2226667404174805 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.222726345062256 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.222426414489746 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.222373008728027 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.222072124481201 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.222296714782715 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.222226619720459 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.222551345825195 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.222671031951904 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.222553253173828 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.222980976104736 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.222517013549805 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.223239898681641 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.222167491912842 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.222917556762695 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.222559452056885 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.222188949584961 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.222656726837158 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.22249174118042 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.221488952636719 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.221717357635498 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.222813129425049 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.222606182098389 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.2225847244262695 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.222578048706055 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.22236967086792 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.222468376159668 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.223493576049805 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.223054885864258 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.223029136657715 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.22343111038208 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.222847938537598 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.22206974029541 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.222682476043701 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.222130298614502 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.223079204559326 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.222491264343262 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.223033905029297 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.222404956817627 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.222252368927002 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.222640514373779 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.222661972045898 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.223004341125488 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.222762107849121 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.222695827484131 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.222914218902588 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.2221879959106445 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.222872257232666 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.22252893447876 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.222872257232666 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.222827911376953 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.222698211669922 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.22275972366333 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.222346782684326 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.222573280334473 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.222734451293945 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.222859859466553 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.22302770614624 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.22271203994751 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.222938537597656 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.2221455574035645 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.222090244293213 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.222546100616455 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.222965717315674 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.222818851470947 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.222565650939941 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.222359657287598 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.22210693359375 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.222203731536865 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.22136926651001 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.223258018493652 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.223047256469727 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.222807884216309 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.223147392272949 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.222341060638428 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.222667694091797 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.222227573394775 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.223099708557129 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.222849369049072 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.223064422607422 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.222999572753906 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.222351551055908 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.22104549407959 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.222567558288574 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.2227253913879395 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.223044395446777 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.2223124504089355 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.222768783569336 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.222656726837158 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.22182035446167 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.2216339111328125 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.2231903076171875 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.22289514541626 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.222068786621094 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.222735404968262 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.223199367523193 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.22199010848999 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.222232341766357 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.2227373123168945 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.222901344299316 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.223334312438965 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.222879886627197 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.22269868850708 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.222206115722656 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.222254753112793 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.222754955291748 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.2226738929748535 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.22313928604126 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.222354412078857 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.2222723960876465 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.221729278564453 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.22173547744751 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.222811698913574 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.22297477722168 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.223220348358154 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.222712993621826 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.222485542297363 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.222193717956543 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.222245216369629 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.222842693328857 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.222750663757324 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.222605228424072 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.222719192504883 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.221617221832275 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.221787929534912 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.221730709075928 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.222864151000977 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.222775459289551 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.222916126251221 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.222999095916748 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.22287130355835 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.222043514251709 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.222858905792236 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.2227606773376465 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.22282600402832 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.222894668579102 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.222779273986816 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.221898078918457 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.222661018371582 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.221452236175537 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.222849369049072 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.222841739654541 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.222413063049316 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.222836017608643 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.221926212310791 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.221818447113037 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.221911907196045 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.2227935791015625 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.222699165344238 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.222606182098389 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.223199844360352 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.222675323486328 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.221879959106445 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.222728252410889 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.22274923324585 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.2226972579956055 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.222929954528809 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.222548007965088 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.222169399261475 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.222088813781738 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.222219467163086 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.2229323387146 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.22268533706665 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.222928524017334 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.222813129425049 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.2227373123168945 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.221445560455322 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.221853733062744 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.222850799560547 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.223044395446777 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.222681045532227 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.222709655761719 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.221596717834473 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.223358631134033 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.223152160644531 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.22278356552124 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.222873210906982 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.222633361816406 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.2230143547058105 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.222998142242432 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.2232279777526855 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.222745895385742 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.222896099090576 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.222933769226074 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.222844123840332 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.222847938537598 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.221916198730469 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.221567630767822 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.221690654754639 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.222659587860107 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.222782135009766 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.2226972579956055 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.222814083099365 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.223433494567871 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.2210845947265625 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.222182750701904 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.222905158996582 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.222850322723389 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.222914218902588 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.222742557525635 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.223117828369141 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.223362445831299 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.2216925621032715 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.222663879394531 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.222630023956299 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.223151206970215 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.222663879394531 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.223068714141846 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.2220025062561035 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.221817493438721 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.223177909851074 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.222588062286377 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.222745418548584 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.2229838371276855 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.222381114959717 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.222255706787109 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.222785472869873 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.222266674041748 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.222801685333252 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.222661972045898 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.222039699554443 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.222270488739014 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.223042964935303 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.22232723236084 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.222636699676514 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.222424507141113 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.220756530761719 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.2226972579956055 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.220427513122559 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.224151134490967 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.223072528839111 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.222182273864746 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.222144603729248 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.221210956573486 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.222382068634033 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.220567226409912 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.22315788269043 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.22350549697876 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.222189903259277 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.223144054412842 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.22215461730957 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.222355365753174 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.221931457519531 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.221909999847412 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.222391128540039 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.22362756729126 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.223321914672852 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.22059965133667 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.222516059875488 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.220762252807617 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.222110271453857 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.219952583312988 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.227688312530518 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.229348659515381 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.220593452453613 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.222654819488525 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.220869064331055 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.2202301025390625 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.223254680633545 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.223137378692627 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.2257513999938965 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.222569465637207 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.221245765686035 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.222594261169434 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.223108768463135 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.224037170410156 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.222744941711426 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.223272800445557 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.221734046936035 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.222740650177002 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.221648693084717 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.223162651062012 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.220844268798828 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.222988128662109 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.222972869873047 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.221874713897705 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.2226738929748535 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.221977233886719 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.223721981048584 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.222413063049316 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.223434925079346 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.223387241363525 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.223112106323242 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.222840309143066 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.2203688621521 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.221194744110107 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.218648910522461 + } + ] + }, + { + "bit": 8, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.102135181427002 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.102354526519775 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.102102756500244 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.101851463317871 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.1021623611450195 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.102006435394287 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.1021647453308105 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.101975917816162 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.101830005645752 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.1019182205200195 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.10241174697876 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.101870536804199 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.101973533630371 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.101809024810791 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.102189064025879 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.102071762084961 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.102224349975586 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.10188102722168 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.101984024047852 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.101903915405273 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.101840019226074 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.101848602294922 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.1021504402160645 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.101745128631592 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.102258205413818 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.101792812347412 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.1022443771362305 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.101404190063477 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.101811408996582 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.102158546447754 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.102097034454346 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.1022748947143555 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.101930618286133 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.101975917816162 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.102043151855469 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.102120876312256 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.1020331382751465 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.102134704589844 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.101778984069824 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.101787090301514 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.101738929748535 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.101794242858887 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.101925849914551 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.102226257324219 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.102242946624756 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.102202415466309 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.101810455322266 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.101748943328857 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.101944446563721 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.102125644683838 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.102341651916504 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.101953506469727 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.101861000061035 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.102115631103516 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.102090835571289 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.102415084838867 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.1020355224609375 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.102106094360352 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.10202169418335 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.101987361907959 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.102029800415039 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.102245330810547 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.101892471313477 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.102365493774414 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.101861000061035 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.102473735809326 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.101811408996582 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.102016925811768 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.102236747741699 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.102318286895752 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.1022748947143555 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.102260589599609 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.101668357849121 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.1022772789001465 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.10234260559082 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.102184772491455 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.101986408233643 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.102006912231445 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.102144718170166 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.1020402908325195 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.102065086364746 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.101884841918945 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.10189962387085 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.101944446563721 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.102165699005127 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.102214336395264 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.101960182189941 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.102359294891357 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.10184907913208 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.101962566375732 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.101799011230469 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.101720333099365 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.10193395614624 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.101840019226074 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.101638317108154 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.102101802825928 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.101911544799805 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.101955890655518 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.102018356323242 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.101937294006348 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.102059841156006 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.102153301239014 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.1023383140563965 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.101970195770264 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.101748943328857 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.102016925811768 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.102136135101318 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.101696014404297 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.101493835449219 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.102640628814697 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.1018266677856445 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.10170841217041 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.102075099945068 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.101939678192139 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.102631568908691 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.101898193359375 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.102247714996338 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.101507186889648 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.10175085067749 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.1020121574401855 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.102073669433594 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.101777076721191 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.101733207702637 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.102269172668457 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.101736068725586 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.101772308349609 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.101892471313477 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.101850509643555 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.101766586303711 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.101526260375977 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.102026462554932 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.1017584800720215 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.1018290519714355 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.101990222930908 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.102039337158203 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.101863861083984 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.101494312286377 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.102043151855469 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.10237979888916 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.101587772369385 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.101956844329834 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.1019978523254395 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.1015825271606445 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.101810455322266 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.101747035980225 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.102138042449951 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.101481914520264 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.102209091186523 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.102026462554932 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.100831508636475 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.1019206047058105 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.101540565490723 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.102360248565674 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.101787090301514 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.101275444030762 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.101767063140869 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.1010823249816895 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.101945400238037 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.101950168609619 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.101991176605225 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.101840972900391 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.1017746925354 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.101735591888428 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.101843357086182 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.102200984954834 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.102293491363525 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.101528644561768 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.1014227867126465 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.102214336395264 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.101597309112549 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.102358818054199 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.101844787597656 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.102240562438965 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.101938247680664 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.1020331382751465 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.101933479309082 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.101769924163818 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.101943492889404 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.101785659790039 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.101810455322266 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.101985454559326 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.1018877029418945 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.102184295654297 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.1020588874816895 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.101816654205322 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.101789474487305 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.101613521575928 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.100895881652832 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.1015424728393555 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.101689338684082 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.101806640625 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.1014404296875 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.1014838218688965 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.101550579071045 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.1020731925964355 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.101269721984863 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.102047443389893 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.101892471313477 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.1017069816589355 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.101562976837158 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.101238250732422 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.101902961730957 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.102085113525391 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.101987361907959 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.1020355224609375 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.101788520812988 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.101792335510254 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.101931571960449 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.101333141326904 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.101712703704834 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.10191011428833 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.1020636558532715 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.1013898849487305 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.101612091064453 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.101259708404541 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.10171365737915 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.10181188583374 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.101771831512451 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.101447582244873 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.10085916519165 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.101559162139893 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.10237979888916 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.101775169372559 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.101405620574951 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.101646423339844 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.10215950012207 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.102007865905762 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.101572513580322 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.101474285125732 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.102765083312988 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.101861953735352 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.10200309753418 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.1018290519714355 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.101386070251465 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.10280179977417 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.101738929748535 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.102015495300293 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.101848602294922 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.101963996887207 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.102303981781006 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.102753639221191 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.102110385894775 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.101373672485352 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.101783275604248 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.102289199829102 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.10225248336792 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.101757049560547 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.101702690124512 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.101684093475342 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.1016340255737305 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.101712703704834 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.10260534286499 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.101606845855713 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.102247714996338 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.102217197418213 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.10215950012207 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.10147762298584 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.101912975311279 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.102314472198486 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.101778984069824 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.102043628692627 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.101078987121582 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.100782871246338 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.1020827293396 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.101583480834961 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.101421356201172 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.101832866668701 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.101983547210693 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.102015495300293 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.101807594299316 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.10141134262085 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.102160930633545 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.10216760635376 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.1018385887146 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.101914882659912 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.101817607879639 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.101783752441406 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.101819038391113 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.102090835571289 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.1012678146362305 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.102115631103516 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.101877212524414 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.101985454559326 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.1018385887146 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.101770401000977 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.100937366485596 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.102040767669678 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.101822376251221 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.1023101806640625 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.102262020111084 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.101808071136475 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.101734161376953 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.10255241394043 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.101819038391113 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.1022629737854 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.102005958557129 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.102168083190918 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.101770401000977 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.101673603057861 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.101622581481934 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.101361274719238 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.102087020874023 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.102052688598633 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.101285457611084 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.102258682250977 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.101741790771484 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.101675510406494 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.10129976272583 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.101907253265381 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.101737022399902 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.101869106292725 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.102607250213623 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.10172700881958 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.101840019226074 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.101797580718994 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.101794242858887 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.1018829345703125 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.102132797241211 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.1020684242248535 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.101680278778076 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.101058006286621 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.101982593536377 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.101973533630371 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.101935863494873 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.102173328399658 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.101921558380127 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.102106094360352 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.1016387939453125 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.10161018371582 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.10191011428833 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.102040767669678 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.102114200592041 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.10206937789917 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.101545333862305 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.10180139541626 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.101696491241455 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.102351188659668 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.101672649383545 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.102272987365723 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.10189151763916 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.101661682128906 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.1014204025268555 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.101696014404297 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.101994514465332 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.10203218460083 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.1020283699035645 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.102179050445557 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.101635456085205 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.101080894470215 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.101515293121338 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.1021013259887695 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.1019206047058105 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.101656913757324 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.101949214935303 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.100675106048584 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.101885795593262 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.100702285766602 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.101758003234863 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.102075576782227 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.102060317993164 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.102001190185547 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.101310729980469 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.101473331451416 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.101884365081787 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.101808547973633 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.102216720581055 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.102541923522949 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.101712703704834 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.101273536682129 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.101531982421875 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.101102828979492 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.102193832397461 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.102074146270752 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.101524353027344 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.101534843444824 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.10112190246582 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.101005554199219 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.100375652313232 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.101830959320068 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.102065086364746 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.101808547973633 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.1021647453308105 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.101684093475342 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.101878643035889 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.101745128631592 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.10213041305542 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.102028846740723 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.101655006408691 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.101809501647949 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.101689338684082 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.101667404174805 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.100645065307617 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.102042198181152 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.102123737335205 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.102062225341797 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.10203218460083 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.101795196533203 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.1015448570251465 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.102048873901367 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.102080345153809 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.1019768714904785 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.10202169418335 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.101901531219482 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.101160526275635 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.101318359375 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.100939750671387 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.10198450088501 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.1020989418029785 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.1022162437438965 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.102085113525391 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.1019487380981445 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.1013569831848145 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.1021246910095215 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.102234840393066 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.102100849151611 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.102107048034668 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.101777076721191 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.101378440856934 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.101544380187988 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.101587772369385 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.101986408233643 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.101888656616211 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.102096080780029 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.101827144622803 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.101506233215332 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.10193395614624 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.100698947906494 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.102088451385498 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.101966857910156 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.10200309753418 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.102042198181152 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.102334499359131 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.101924419403076 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.101462364196777 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.101926326751709 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.101998805999756 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.10202169418335 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.101807117462158 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.101317405700684 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.102512836456299 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.101623058319092 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.102049827575684 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.102079391479492 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.101982593536377 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.102005958557129 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.10219144821167 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.100962162017822 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.101125240325928 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.102043151855469 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.102184295654297 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.101868629455566 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.101751804351807 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.1011962890625 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.1009321212768555 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.101925849914551 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.1019721031188965 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.102174758911133 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.101804733276367 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.101702690124512 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.1021552085876465 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.1021552085876465 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.101585388183594 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.1020073890686035 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.1020307540893555 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.102199554443359 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.101942539215088 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.100917816162109 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.101807594299316 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.101922035217285 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.102117538452148 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.1019978523254395 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.102421283721924 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.102287769317627 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.101183891296387 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.10092306137085 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.101989269256592 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.101842880249023 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.101926803588867 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.10148286819458 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.101806640625 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.10080623626709 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.101820945739746 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.101781368255615 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.102197647094727 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.101692199707031 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.1019978523254395 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.101830005645752 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.10175895690918 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.101132869720459 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.101306915283203 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.1019768714904785 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.102056503295898 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.102121353149414 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.101736068725586 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.100907802581787 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.102255344390869 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.101115703582764 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.101827144622803 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.10154390335083 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.09985876083374 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.102015495300293 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.100078582763672 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.103763580322266 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.100894451141357 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.101039886474609 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.101334095001221 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.101462364196777 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.1012349128723145 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.10159158706665 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.102100849151611 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.101275444030762 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.101569175720215 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.10206937789917 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.100222587585449 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.101302146911621 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.101271152496338 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.10261344909668 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.102534770965576 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.101774215698242 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.102444171905518 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.10183572769165 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.1016364097595215 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.09926176071167 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.102370738983154 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.100998878479004 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.105584144592285 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.104708671569824 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.101344108581543 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.101873874664307 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.099835395812988 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.1043500900268555 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.101168155670166 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.10276985168457 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.1029157638549805 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.100982189178467 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.101600170135498 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.101637363433838 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.102969646453857 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.102789878845215 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.101871967315674 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.102039813995361 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.10125207901001 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.101796627044678 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.101291656494141 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.102401256561279 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.102728843688965 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.102071762084961 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.101532936096191 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.10213041305542 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.101640224456787 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.100052833557129 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.103302955627441 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.102257251739502 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.102384567260742 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.102415561676025 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.102062702178955 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.102531433105469 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.100672245025635 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.100336074829102 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.09989595413208 + } + ] + }, + { + "bit": 9, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 4.013276100158691 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 4.013246059417725 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 4.013744354248047 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 4.013569355010986 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 4.013381481170654 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 4.01316499710083 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 4.01309061050415 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 4.013267517089844 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 4.0132927894592285 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 4.013316631317139 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 4.013128280639648 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 4.013118267059326 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 4.013436317443848 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 4.013391017913818 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 4.013280868530273 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 4.013282775878906 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 4.013035297393799 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 4.013152122497559 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 4.013113021850586 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 4.013160705566406 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 4.013382434844971 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 4.013089656829834 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 4.013437271118164 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 4.0134711265563965 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 4.01287317276001 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 4.013000965118408 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 4.01287317276001 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 4.013498306274414 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 4.013105392456055 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 4.013362407684326 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 4.013190269470215 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 4.013127326965332 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 4.013180255889893 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 4.013104438781738 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 4.013342380523682 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 4.0132646560668945 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 4.013130187988281 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 4.013127326965332 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 4.013333797454834 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 4.013293743133545 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 4.013102054595947 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 4.013253211975098 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 4.013411998748779 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 4.013352870941162 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 4.013067245483398 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 4.013220310211182 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 4.013350963592529 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 4.013332843780518 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 4.013114929199219 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 4.013484477996826 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 4.013267517089844 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 4.013426780700684 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 4.013246059417725 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 4.012973785400391 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 4.013215065002441 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 4.013249397277832 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 4.0134053230285645 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 4.013198375701904 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 4.013156414031982 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 4.013200759887695 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 4.013349533081055 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 4.013309955596924 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 4.013247489929199 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 4.0131449699401855 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 4.013466835021973 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 4.012912273406982 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 4.013105869293213 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 4.0131330490112305 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 4.013261318206787 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 4.012759685516357 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 4.013085842132568 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 4.01332950592041 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 4.0132904052734375 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 4.01305627822876 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 4.013186931610107 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 4.01311731338501 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 4.012913227081299 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 4.0131707191467285 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 4.013248443603516 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 4.013122081756592 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 4.013216972351074 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 4.0131964683532715 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 4.013406753540039 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 4.013045310974121 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 4.013362407684326 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 4.013455867767334 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 4.013102054595947 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 4.013301372528076 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 4.013550758361816 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 4.013108730316162 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 4.013391017913818 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 4.013164043426514 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 4.013036727905273 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 4.013116359710693 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 4.013236999511719 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 4.013319969177246 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 4.013136386871338 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 4.013235092163086 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 4.013325214385986 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 4.013041019439697 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 4.0132646560668945 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 4.013000965118408 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 4.013250350952148 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 4.013331890106201 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 4.013766288757324 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 4.013393878936768 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 4.013017177581787 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 4.01318359375 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 4.013442039489746 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 4.013190269470215 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 4.013293743133545 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 4.013357639312744 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 4.013003349304199 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 4.013354301452637 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 4.013139247894287 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 4.01342248916626 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 4.013237476348877 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 4.013526439666748 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 4.013501167297363 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 4.013258934020996 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 4.013307094573975 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 4.013250350952148 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 4.012836933135986 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 4.013187885284424 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 4.013595104217529 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 4.013187885284424 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 4.013011932373047 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 4.013269424438477 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 4.012774467468262 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 4.013088226318359 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 4.013265609741211 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 4.012978553771973 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 4.013375759124756 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 4.013289451599121 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 4.013316631317139 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 4.0128278732299805 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 4.0130934715271 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 4.013206481933594 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 4.013012886047363 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 4.012919902801514 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 4.013375759124756 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 4.013075351715088 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 4.013269424438477 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 4.0131425857543945 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 4.013097763061523 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 4.013309001922607 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 4.013062953948975 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 4.013332843780518 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 4.01294469833374 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 4.013313293457031 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 4.0131120681762695 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 4.013236999511719 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 4.012994766235352 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 4.0132904052734375 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 4.013075351715088 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 4.01301383972168 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 4.013044357299805 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 4.012532711029053 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 4.012791156768799 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 4.013277530670166 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 4.013047695159912 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 4.013067722320557 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 4.013186931610107 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 4.012531280517578 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 4.012821197509766 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 4.013017177581787 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 4.013773441314697 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 4.013557434082031 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 4.013113021850586 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 4.013657093048096 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 4.012439250946045 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 4.012542724609375 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 4.012807369232178 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 4.012981414794922 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 4.013309955596924 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 4.013098239898682 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 4.0131072998046875 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 4.013246059417725 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 4.013294696807861 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 4.01272439956665 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 4.0130205154418945 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 4.013123035430908 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 4.013216018676758 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 4.013382434844971 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 4.0133891105651855 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 4.0127997398376465 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 4.013055324554443 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 4.01370906829834 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 4.013634204864502 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 4.012934684753418 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 4.012977123260498 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 4.013434410095215 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 4.0128984451293945 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 4.013298988342285 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 4.0128936767578125 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 4.013464450836182 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 4.0129618644714355 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 4.013092994689941 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 4.013060569763184 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 4.012823104858398 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 4.0132832527160645 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 4.013060092926025 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 4.0129923820495605 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 4.013392925262451 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 4.013211727142334 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 4.013205528259277 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 4.012814044952393 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 4.012908458709717 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 4.013241291046143 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 4.012581825256348 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 4.013176918029785 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 4.013042449951172 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 4.012859344482422 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 4.0135579109191895 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 4.013217926025391 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 4.012973785400391 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 4.012877941131592 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 4.013132095336914 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 4.012957572937012 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 4.013545989990234 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 4.013316631317139 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 4.012685298919678 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 4.013335227966309 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 4.013518333435059 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 4.0131425857543945 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 4.013327121734619 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 4.012738227844238 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 4.013293743133545 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 4.013156414031982 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 4.0125603675842285 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 4.012983322143555 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 4.013465881347656 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 4.013185024261475 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 4.013145923614502 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 4.0124735832214355 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 4.013175010681152 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 4.013239860534668 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 4.0133538246154785 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 4.013408660888672 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 4.013260841369629 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 4.012509346008301 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 4.012801647186279 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 4.01289176940918 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 4.012633800506592 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 4.0127339363098145 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 4.013525009155273 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 4.013258934020996 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 4.012931823730469 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 4.012784481048584 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 4.013160705566406 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 4.013242721557617 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 4.01246976852417 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 4.013128757476807 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 4.013041019439697 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 4.012689590454102 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 4.01249885559082 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 4.013518810272217 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 4.013340473175049 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 4.0127081871032715 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 4.013350486755371 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 4.012989044189453 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 4.013739109039307 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 4.0132904052734375 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 4.012703895568848 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 4.012840270996094 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 4.013317584991455 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 4.013212203979492 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 4.013317584991455 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 4.0129570960998535 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 4.013494491577148 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 4.013015270233154 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 4.012806415557861 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 4.011937618255615 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 4.013210773468018 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 4.013381481170654 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 4.013113498687744 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 4.013238906860352 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 4.012447834014893 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 4.012640953063965 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 4.013239860534668 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 4.013152122497559 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 4.013401508331299 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 4.013125419616699 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 4.01344633102417 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 4.013202667236328 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 4.013721942901611 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 4.012469291687012 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 4.0134053230285645 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 4.013370037078857 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 4.012970447540283 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 4.012993335723877 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 4.012542724609375 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 4.012766361236572 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 4.012108325958252 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 4.013250350952148 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 4.013187885284424 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 4.013289451599121 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 4.0130510330200195 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 4.013066291809082 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 4.012284278869629 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 4.012354850769043 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 4.013678073883057 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 4.013461589813232 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 4.013828277587891 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 4.013082027435303 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 4.012794017791748 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 4.012677192687988 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 4.013011932373047 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 4.013311386108398 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 4.013147830963135 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 4.013163089752197 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 4.013152599334717 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 4.013036727905273 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 4.012741565704346 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 4.0121073722839355 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 4.013180255889893 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 4.012996673583984 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 4.013047695159912 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 4.012936592102051 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 4.012837886810303 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 4.012994289398193 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 4.012278079986572 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 4.013298988342285 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 4.013049602508545 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 4.012935638427734 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 4.013134956359863 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 4.012934684753418 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 4.013202667236328 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 4.013150215148926 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 4.013296127319336 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 4.013223648071289 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 4.013051509857178 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 4.013055324554443 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 4.012567520141602 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 4.012401580810547 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 4.013019561767578 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 4.013343334197998 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 4.013176918029785 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 4.012900352478027 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 4.013256549835205 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 4.012588977813721 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 4.013326168060303 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 4.01254940032959 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 4.0135087966918945 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 4.013214111328125 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 4.0131096839904785 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 4.013033390045166 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 4.012835502624512 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 4.013453006744385 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 4.012712478637695 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 4.013047695159912 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 4.0133891105651855 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 4.013411521911621 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 4.01315450668335 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 4.013810157775879 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 4.012242794036865 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 4.013425350189209 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 4.013331890106201 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 4.013049602508545 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 4.013195514678955 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 4.013330936431885 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 4.012334823608398 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 4.0129265785217285 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 4.012275695800781 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 4.013263702392578 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 4.013164043426514 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 4.012907028198242 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 4.013230323791504 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 4.0132904052734375 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 4.013017177581787 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 4.012618064880371 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 4.013439655303955 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 4.0131072998046875 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 4.0133795738220215 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 4.013303756713867 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 4.012998580932617 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 4.013119697570801 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 4.01257848739624 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 4.013347148895264 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 4.013166427612305 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 4.013233184814453 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 4.013298034667969 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 4.012704849243164 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 4.012507915496826 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 4.012803554534912 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 4.013177871704102 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 4.013193130493164 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 4.013594150543213 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 4.013371467590332 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 4.013000965118408 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 4.01228141784668 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 4.013082027435303 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 4.01319694519043 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 4.013180255889893 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 4.012876987457275 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 4.012869358062744 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 4.012594699859619 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 4.012807846069336 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 4.012391567230225 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 4.013286590576172 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 4.013139247894287 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 4.013219356536865 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 4.013191223144531 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 4.012711524963379 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 4.0128607749938965 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 4.013057231903076 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 4.013092517852783 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 4.013169765472412 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 4.012979030609131 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 4.012844562530518 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 4.012796878814697 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 4.013040065765381 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 4.01296329498291 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 4.013136386871338 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 4.01317834854126 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 4.013033390045166 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 4.013291358947754 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 4.013003349304199 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 4.012793064117432 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 4.012574195861816 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 4.013179302215576 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 4.013035297393799 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 4.013279914855957 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 4.013205528259277 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 4.012751579284668 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 4.012217998504639 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 4.012819290161133 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 4.013156414031982 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 4.013303279876709 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 4.013458251953125 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 4.012851715087891 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 4.01259183883667 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 4.01349401473999 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 4.013036727905273 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 4.012936592102051 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 4.013088226318359 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 4.013112545013428 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 4.012948513031006 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 4.01301383972168 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 4.012267112731934 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 4.012787342071533 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 4.013003349304199 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 4.013128757476807 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 4.013057708740234 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 4.0130486488342285 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 4.012668609619141 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 4.013030052185059 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 4.013545989990234 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 4.013004302978516 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 4.012989044189453 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 4.0128493309021 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 4.012850761413574 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 4.012856483459473 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 4.013023376464844 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 4.012651443481445 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 4.013132572174072 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 4.012962341308594 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 4.013025760650635 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 4.013288497924805 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 4.012739658355713 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 4.01182222366333 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 4.012296676635742 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 4.013134956359863 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 4.012942314147949 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 4.013004302978516 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 4.012816905975342 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 4.012372016906738 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 4.012099742889404 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 4.012835502624512 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 4.013113498687744 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 4.013172626495361 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 4.013107776641846 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 4.012990951538086 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 4.012612342834473 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 4.013084411621094 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 4.0128493309021 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 4.013214588165283 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 4.013025760650635 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 4.013278961181641 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 4.01299524307251 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 4.012984752655029 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 4.0124053955078125 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 4.012167453765869 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 4.01338529586792 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 4.013129234313965 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 4.012889862060547 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 4.01304292678833 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 4.012568950653076 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 4.012996673583984 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 4.012847423553467 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 4.013090133666992 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 4.013069152832031 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 4.012742519378662 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 4.0125203132629395 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 4.012726783752441 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 4.013007164001465 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 4.012365341186523 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 4.012855052947998 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 4.013111591339111 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 4.010582447052002 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 4.012889862060547 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 4.011747360229492 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 4.014633655548096 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 4.013059616088867 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 4.013525009155273 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 4.01345682144165 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 4.011638641357422 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 4.01312780380249 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 4.012121677398682 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 4.0126824378967285 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 4.013805866241455 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 4.012590408325195 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 4.013416767120361 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 4.012486457824707 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 4.0122575759887695 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 4.012590408325195 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 4.012233734130859 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 4.012847900390625 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 4.013658046722412 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 4.013572692871094 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 4.01049280166626 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 4.013051509857178 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 4.011860370635986 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 4.012496471405029 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 4.01137638092041 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 4.013706684112549 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 4.016254901885986 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 4.0103349685668945 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 4.012597560882568 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 4.012121677398682 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 4.012214183807373 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 4.013614654541016 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 4.012634754180908 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 4.0153489112854 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 4.01272439956665 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 4.0121612548828125 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 4.012907981872559 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 4.013936996459961 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 4.014367580413818 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 4.013075351715088 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 4.013566017150879 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 4.012102127075195 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 4.013122081756592 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 4.0122857093811035 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 4.013699054718018 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 4.012650489807129 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 4.013038635253906 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 4.013289451599121 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 4.01247501373291 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 4.013031482696533 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 4.01260232925415 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 4.013372898101807 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 4.0134992599487305 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 4.013482093811035 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 4.013452529907227 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 4.013139247894287 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 4.0132951736450195 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 4.011699676513672 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 4.011867523193359 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 4.008990287780762 + } + ] + }, + { + "bit": 10, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.9282772541046143 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.928635597229004 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.928539514541626 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.9285900592803955 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.928414821624756 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.928699016571045 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.928347110748291 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.928661346435547 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.928617477416992 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.9287235736846924 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.928680181503296 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.928593158721924 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.9284794330596924 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.9284496307373047 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.9284801483154297 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.9284145832061768 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.928896188735962 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.9283618927001953 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.9285359382629395 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.928333282470703 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.928421974182129 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.9286389350891113 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.928352117538452 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.9286632537841797 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.928539514541626 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.9285900592803955 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.928680181503296 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.9281818866729736 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.928689479827881 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.9286506175994873 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.9285478591918945 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.928323268890381 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.9285330772399902 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.9281325340270996 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.9280877113342285 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.928609848022461 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.928633213043213 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.92830491065979 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.928510904312134 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.92840838432312 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.9287190437316895 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.928323268890381 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.9284451007843018 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.9283699989318848 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.9286770820617676 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.9284729957580566 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.9286627769470215 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.928354501724243 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.928778648376465 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.9285342693328857 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.9284589290618896 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.928520679473877 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.9282689094543457 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.9286885261535645 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.9283652305603027 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.928457021713257 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.928410768508911 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.9286181926727295 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.9281997680664062 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.9290173053741455 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.928290843963623 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.92840576171875 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.928279399871826 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.928528070449829 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.9285519123077393 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.9287407398223877 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.928462028503418 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.9282896518707275 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.928720235824585 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.9288113117218018 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.9290575981140137 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.928744316101074 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.928753137588501 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.928621530532837 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.9286913871765137 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.9286274909973145 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.928558588027954 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.928687334060669 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.9285647869110107 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.9285666942596436 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.9286725521087646 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.928377628326416 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.928579807281494 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.928720235824585 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.9286274909973145 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.9286410808563232 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.928401470184326 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.9287614822387695 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.92842435836792 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.928367853164673 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.928663492202759 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.928556442260742 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.928516149520874 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.92863130569458 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.9283690452575684 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.9286715984344482 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.928553581237793 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.928560972213745 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.928514242172241 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.928382158279419 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.928553581237793 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.9289498329162598 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.9284234046936035 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.928739070892334 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.9284064769744873 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.9285783767700195 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.928518056869507 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.9283382892608643 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.928199052810669 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.928544759750366 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.928830146789551 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.9284441471099854 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.9287962913513184 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.928689479827881 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.928905487060547 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.928447723388672 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.928731679916382 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.9282052516937256 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.9284234046936035 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.928414821624756 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.9287943840026855 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.9284794330596924 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.928433418273926 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.9288365840911865 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.9287118911743164 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.92875337600708 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.9283194541931152 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.9286537170410156 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.928591251373291 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.9279987812042236 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.928504705429077 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.928661346435547 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.928528070449829 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.928520679473877 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.9285881519317627 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.9285712242126465 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.9286491870880127 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.928614616394043 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.9283080101013184 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.928151845932007 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.928617477416992 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.928347587585449 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.928553581237793 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.9284873008728027 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.9286739826202393 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.928497314453125 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.928924798965454 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.928837776184082 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.92863130569458 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.9274771213531494 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.928626537322998 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.928403854370117 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.928830623626709 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.928375482559204 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.9283552169799805 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.928293228149414 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.9278054237365723 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.9280498027801514 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.9288060665130615 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.9285316467285156 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.928417682647705 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.9283194541931152 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.928483724594116 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.927715539932251 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.9278953075408936 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.9287962913513184 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.9283447265625 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.927922487258911 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.9286651611328125 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.9284634590148926 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.9286715984344482 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.928518056869507 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.928818464279175 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.928664207458496 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.929006338119507 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.928433656692505 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.9283812046051025 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.928262233734131 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.9282877445220947 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.928347587585449 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.928492784500122 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.927856206893921 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.928736448287964 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.9286303520202637 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.9283499717712402 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.9288337230682373 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.9284863471984863 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.9278435707092285 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.9279448986053467 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.9284768104553223 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.9286434650421143 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.927457809448242 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.9284989833831787 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.9284181594848633 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.9286048412323 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.9282071590423584 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.928410768508911 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.928654432296753 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.9290854930877686 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.928731679916382 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.9280197620391846 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.9283924102783203 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.92840576171875 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.9284369945526123 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.928252935409546 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.9285871982574463 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.928892135620117 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.9286420345306396 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.928262233734131 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.928389549255371 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.9284350872039795 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.928361654281616 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.928058385848999 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.928255081176758 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.9282896518707275 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.9282174110412598 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.9281129837036133 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.92828631401062 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.928388833999634 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.927961826324463 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.927961826324463 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.92857027053833 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.9284579753875732 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.9279394149780273 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.92858624458313 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.9284768104553223 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.9288580417633057 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.9278695583343506 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.9282760620117188 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.9287381172180176 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.9282424449920654 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.9285807609558105 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.928826093673706 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.9285459518432617 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.9286842346191406 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.928295135498047 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.9286837577819824 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.928443431854248 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.9284534454345703 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.928661823272705 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.928830146789551 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.928550958633423 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.9275777339935303 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.9286999702453613 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.9288017749786377 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.9284749031066895 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.9284873008728027 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.9282689094543457 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.9286444187164307 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.928659200668335 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.9283249378204346 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.9285037517547607 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.928367853164673 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.9287290573120117 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.928126573562622 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.9283831119537354 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.9280498027801514 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.928285598754883 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.9285199642181396 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.92830753326416 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.9286274909973145 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.9280779361724854 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.9279520511627197 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.9287736415863037 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.928412675857544 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.92769193649292 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.928394317626953 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.9284560680389404 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.928373336791992 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.9286112785339355 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.9282240867614746 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.928600549697876 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.9288134574890137 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.9284191131591797 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.9286417961120605 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.9283204078674316 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.928558588027954 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.9287047386169434 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.9282567501068115 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.9276626110076904 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.9284908771514893 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.9283735752105713 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.9287109375 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.928433656692505 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.9283995628356934 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.9278738498687744 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.9286348819732666 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.9285311698913574 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.928680419921875 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.9286866188049316 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.928145170211792 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.9285519123077393 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.928361654281616 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.9286141395568848 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.928439140319824 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.9286677837371826 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.928394317626953 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.928217887878418 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.9283087253570557 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.928621530532837 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.928551197052002 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.9285635948181152 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.9286084175109863 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.928394317626953 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.928751230239868 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.9284188747406006 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.928266763687134 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.927743673324585 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.9283671379089355 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.9283721446990967 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.9282913208007812 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.9287467002868652 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.9279165267944336 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.92840576171875 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.9281954765319824 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.9284181594848633 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.928489923477173 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.928527593612671 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.9284350872039795 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.9281039237976074 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.9276955127716064 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.928867816925049 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.9286537170410156 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.9285778999328613 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.928593158721924 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.9286739826202393 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.9285967350006104 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.928126573562622 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.9286508560180664 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.9285149574279785 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.92850661277771 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.9285550117492676 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.928727865219116 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.9281561374664307 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.9286696910858154 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.928152561187744 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.928556203842163 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.9286069869995117 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.9287829399108887 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.928544759750366 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.9287259578704834 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.9279720783233643 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.9285619258880615 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.92859148979187 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.928670644760132 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.928797483444214 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.9286770820617676 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.9280335903167725 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.9277923107147217 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.9283366203308105 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.9284603595733643 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.9286715984344482 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.9284720420837402 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.928236722946167 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.927948474884033 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.928515911102295 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.9280104637145996 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.92844820022583 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.9284768104553223 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.9284863471984863 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.9282846450805664 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.9283785820007324 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.9282190799713135 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.9288265705108643 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.928455114364624 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.928663492202759 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.928366184234619 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.928497314453125 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.9282379150390625 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.928325653076172 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.9279088973999023 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.928546905517578 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.928788900375366 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.9286229610443115 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.9285900592803955 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.9279847145080566 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.9281692504882812 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.928123712539673 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.9284048080444336 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.928446054458618 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.92858624458313 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.928553581237793 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.928571939468384 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.9282727241516113 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.928630828857422 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.9287540912628174 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.9285225868225098 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.92846417427063 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.92818546295166 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.928018808364868 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.928593635559082 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.9280929565429688 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.9286983013153076 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.928609848022461 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.928849458694458 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.9285244941711426 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.928527593612671 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.9279637336730957 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.92880916595459 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.9283699989318848 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.9283597469329834 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.928347587585449 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.92850661277771 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.9284656047821045 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.928004026412964 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.9282946586608887 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.928567886352539 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.9285478591918945 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.9285290241241455 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.928528070449829 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.9285783767700195 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.9285995960235596 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.9280242919921875 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.9286603927612305 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.9285638332366943 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.9284656047821045 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.928379535675049 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.928136110305786 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.9282286167144775 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.9286201000213623 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.928508758544922 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.9282097816467285 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.9285213947296143 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.9284167289733887 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.9282801151275635 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.928684711456299 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.9278128147125244 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.928327798843384 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.9284989833831787 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.9282820224761963 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.92842173576355 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.928410291671753 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.928091049194336 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.928234100341797 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.928455114364624 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.928481101989746 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.9284920692443848 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.9286112785339355 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.92798113822937 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.928206443786621 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.9281058311462402 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.928377628326416 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.9285366535186768 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.9284629821777344 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.9284508228302 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.9280948638916016 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.9279234409332275 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.9275903701782227 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.928508758544922 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.9285404682159424 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.9285528659820557 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.9284005165100098 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.9283223152160645 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.9282379150390625 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.928046464920044 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.928475856781006 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.9284422397613525 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.928422451019287 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.9283287525177 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.9284346103668213 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.928680181503296 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.92858624458313 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.9284422397613525 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.928446054458618 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.928426504135132 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.9284956455230713 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.9286537170410156 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.9282708168029785 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.9277231693267822 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.9285223484039307 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.928377151489258 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.928675889968872 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.9286201000213623 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.9286205768585205 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.927809953689575 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.928274631500244 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.9284234046936035 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.928462028503418 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.928377866744995 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.9283978939056396 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.927942991256714 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.9282665252685547 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.9282114505767822 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.9286365509033203 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.928429365158081 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.9280898571014404 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.9284005165100098 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.9283320903778076 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.9279115200042725 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.9280452728271484 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.9284281730651855 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.9284253120422363 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.9282455444335938 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.928286552429199 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.928257703781128 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.9288618564605713 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.9275736808776855 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.928366184234619 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.928086757659912 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.9241392612457275 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.92850661277771 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.927255630493164 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.928936004638672 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.9277336597442627 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.927963972091675 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.9279441833496094 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.927213430404663 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.9277844429016113 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.9278509616851807 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.928288698196411 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.928144693374634 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.9280736446380615 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.9285502433776855 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.9265968799591064 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.927830696105957 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.9275872707366943 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.928737163543701 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.928424835205078 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.9282524585723877 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.9286282062530518 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.927584648132324 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.9280197620391846 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.9265689849853516 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.9290919303894043 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.9285712242126465 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.931112051010132 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.9293582439422607 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.9255380630493164 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.9283182621002197 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.9269814491271973 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.9303359985351562 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.9272847175598145 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.929049253463745 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.928865432739258 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.9275200366973877 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.928162097930908 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.9282350540161133 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.9279987812042236 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.9294629096984863 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.9283053874969482 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.9281489849090576 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.9272501468658447 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.9280076026916504 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.9285078048706055 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.9285528659820557 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.929283380508423 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.928635835647583 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.9279227256774902 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.9283299446105957 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.927931547164917 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.927556037902832 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.929001569747925 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.928584575653076 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.928703546524048 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.9286632537841797 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.92846941947937 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.928616523742676 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.9277327060699463 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.9277734756469727 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.9265427589416504 + } + ] + }, + { + "bit": 11, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.8681704998016357 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.8677256107330322 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.867805242538452 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.867733955383301 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.868157148361206 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.8680806159973145 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.867917537689209 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.8681302070617676 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.868032693862915 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.8681130409240723 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.8677237033843994 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.8678996562957764 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.8679027557373047 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.867894411087036 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.8680078983306885 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.867985963821411 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.8679864406585693 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.86753511428833 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.8679020404815674 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.8682336807250977 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.867845296859741 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.8679182529449463 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.867868185043335 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.867847442626953 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.867980718612671 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.86793851852417 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.8677687644958496 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.867760181427002 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.8681068420410156 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.8680498600006104 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.8678131103515625 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.867920160293579 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.8677172660827637 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.8678057193756104 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.867849826812744 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.8680202960968018 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.867953062057495 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.8680732250213623 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.8678226470947266 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.8679423332214355 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.868065595626831 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.86769437789917 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.868023633956909 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.8679537773132324 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.8676986694335938 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.8679962158203125 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.8678371906280518 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.8677895069122314 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.8674590587615967 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.868021249771118 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.8679680824279785 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.8678953647613525 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.868009328842163 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.867889165878296 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.867570161819458 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.8681986331939697 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.867931604385376 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.8679027557373047 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.868204116821289 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.8678274154663086 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.8681371212005615 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.8678531646728516 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.8680334091186523 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.868136167526245 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.8679676055908203 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.8679726123809814 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.86765718460083 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.8682353496551514 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.868175506591797 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.8674800395965576 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.867967128753662 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.8679587841033936 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.8678324222564697 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.867912530899048 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.867932081222534 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.8678507804870605 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.8678224086761475 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.867917537689209 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.867885112762451 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.8679466247558594 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.8680925369262695 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.868178129196167 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.868053913116455 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.867713451385498 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.868168592453003 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.8681769371032715 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.8680078983306885 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.8678250312805176 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.8679349422454834 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.8678414821624756 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.868108034133911 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.867621421813965 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.867795944213867 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.8679144382476807 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.8682453632354736 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.86789870262146 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.8679661750793457 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.8682258129119873 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.8682796955108643 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.8680381774902344 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.8680808544158936 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.8675010204315186 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.867807626724243 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.8679256439208984 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.8679559230804443 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.8679161071777344 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.867929220199585 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.8680026531219482 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.8680813312530518 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.8677916526794434 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.867605686187744 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.8677382469177246 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.868145227432251 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.8676371574401855 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.8677494525909424 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.867896795272827 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.8678348064422607 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.867750406265259 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.8679404258728027 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.868114471435547 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.867830753326416 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.8677854537963867 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.8680477142333984 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.8678786754608154 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.8678488731384277 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.867830991744995 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.867680788040161 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.8681745529174805 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.867783784866333 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.8680169582366943 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.867889642715454 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.868180990219116 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.868137836456299 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.8680849075317383 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.867994785308838 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.867746591567993 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.86769437789917 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.8677382469177246 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.867941379547119 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.867783784866333 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.8679118156433105 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.8679568767547607 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.8677923679351807 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.8679819107055664 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.8676185607910156 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.8685150146484375 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.8677642345428467 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.868040084838867 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.867600917816162 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.8676226139068604 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.867673397064209 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.8679840564727783 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.8681600093841553 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.868051528930664 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.8677093982696533 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.867696523666382 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.867992401123047 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.867391586303711 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.8673288822174072 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.8677802085876465 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.8680436611175537 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.8679234981536865 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.867816686630249 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.866532325744629 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.867793321609497 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.8673527240753174 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.868265151977539 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.868133068084717 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.8678853511810303 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.8680808544158936 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.867431163787842 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.8678371906280518 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.8671717643737793 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.8679041862487793 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.8672850131988525 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.8677687644958496 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.8681108951568604 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.8678159713745117 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.8681931495666504 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.867743730545044 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.86765718460083 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.8681912422180176 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.8679404258728027 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.8678371906280518 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.868088960647583 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.8674991130828857 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.8676624298095703 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.8684191703796387 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.8679256439208984 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.8679516315460205 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.868184804916382 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.8682215213775635 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.867478132247925 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.867659330368042 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.8676812648773193 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.86787486076355 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.867889165878296 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.867715358734131 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.8675944805145264 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.867739677429199 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.868333101272583 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.867957830429077 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.867548942565918 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.8680169582366943 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.867779016494751 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.8679208755493164 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.867349863052368 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.8679306507110596 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.868424654006958 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.8679020404815674 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.8679163455963135 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.8678441047668457 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.8676156997680664 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.8677124977111816 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.8676812648773193 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.8681046962738037 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.8679559230804443 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.867894411087036 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.8675529956817627 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.867668390274048 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.867926597595215 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.867678642272949 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.867880344390869 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.8676841259002686 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.8678841590881348 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.8677611351013184 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.8674166202545166 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.868312120437622 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.8681228160858154 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.8670969009399414 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.8681108951568604 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.8679921627044678 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.867941379547119 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.8680474758148193 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.8674724102020264 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.868145227432251 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.8679957389831543 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.8677499294281006 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.8679921627044678 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.8680598735809326 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.867021083831787 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.867374897003174 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.8677237033843994 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.8676629066467285 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.8674728870391846 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.8680245876312256 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.8676443099975586 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.8677480220794678 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.867526054382324 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.8678245544433594 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.8681790828704834 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.8674776554107666 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.8678488731384277 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.8679051399230957 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.8679697513580322 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.8671464920043945 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.8684945106506348 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.8680243492126465 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.867243766784668 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.8678619861602783 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.867949962615967 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.867802858352661 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.868196725845337 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.867666482925415 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.8682477474212646 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.8680975437164307 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.8678205013275146 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.8679349422454834 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.8680531978607178 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.867765426635742 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.8680217266082764 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.867643356323242 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.867175579071045 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.8677268028259277 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.8679163455963135 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.867842435836792 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.8677682876586914 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.8675668239593506 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.8675925731658936 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.8680925369262695 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.867802619934082 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.8680436611175537 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.867678642272949 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.8675613403320312 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.8680830001831055 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.8682363033294678 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.8672657012939453 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.8680477142333984 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.867978096008301 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.8679301738739014 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.868102788925171 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.867830991744995 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.8677446842193604 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.8674683570861816 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.8677661418914795 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.8679285049438477 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.8679912090301514 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.867741584777832 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.867460012435913 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.8673696517944336 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.8677237033843994 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.868016004562378 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.868051052093506 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.8681697845458984 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.8677818775177 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.8678090572357178 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.8679487705230713 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.867898464202881 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.8682215213775635 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.8679070472717285 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.868061065673828 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.867934226989746 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.868229866027832 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.8677268028259277 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.8671209812164307 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.867870330810547 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.867842435836792 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.8679027557373047 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.868086099624634 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.8676822185516357 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.8679957389831543 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.867241859436035 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.8679070472717285 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.8680758476257324 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.867863893508911 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.867719888687134 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.8675663471221924 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.8681395053863525 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.867647886276245 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.867997646331787 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.867908000946045 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.8680450916290283 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.8678901195526123 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.868058204650879 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.867147445678711 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.867656946182251 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.867889642715454 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.867765188217163 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.8677895069122314 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.86795973777771 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.8676259517669678 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.8681633472442627 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.867500066757202 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.868008852005005 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.8678693771362305 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.86789870262146 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.867774486541748 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.867565155029297 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.867978096008301 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.867626428604126 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.8680925369262695 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.8678934574127197 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.8678739070892334 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.8680906295776367 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.8682146072387695 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.8670907020568848 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.8673288822174072 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.8678665161132812 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.8678112030029297 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.8677687644958496 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.868215322494507 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.8678269386291504 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.86779522895813 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.867475748062134 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.867957353591919 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.8680267333984375 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.867870330810547 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.867741584777832 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.8678619861602783 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.867743730545044 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.8681821823120117 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.867905855178833 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.8677818775177 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.8678081035614014 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.8678927421569824 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.867868185043335 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.8677475452423096 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.8672749996185303 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.867833375930786 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.8679518699645996 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.867868185043335 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.8677563667297363 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.8672780990600586 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.867258071899414 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.867610216140747 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.867814064025879 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.8678953647613525 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.867718458175659 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.867992877960205 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.8678674697875977 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.867199659347534 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.867550849914551 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.867851495742798 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.8680362701416016 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.867901563644409 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.867868423461914 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.8671298027038574 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.867469310760498 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.8670551776885986 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.8678786754608154 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.8679301738739014 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.8679559230804443 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.867957830429077 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.8679680824279785 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.867872953414917 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.8680572509765625 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.867979049682617 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.8680005073547363 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.8678975105285645 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.8680498600006104 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.8672497272491455 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.86753511428833 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.867846727371216 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.867903470993042 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.867926597595215 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.867830991744995 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.867798328399658 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.8680005073547363 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.8676843643188477 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.867259979248047 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.8679134845733643 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.8679957389831543 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.867997646331787 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.8679745197296143 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.867156505584717 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.867516279220581 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.8679895401000977 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.867786407470703 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.8678317070007324 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.8680598735809326 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.867920160293579 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.8677587509155273 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.8675901889801025 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.8679425716400146 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.867943286895752 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.867903470993042 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.8679375648498535 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.867733955383301 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.868079662322998 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.8674259185791016 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.867795467376709 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.867954969406128 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.8679192066192627 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.8678383827209473 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.867704153060913 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.867677688598633 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.8681228160858154 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.8681507110595703 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.867753028869629 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.867763042449951 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.867544651031494 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.86785626411438 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.8679144382476807 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.8676812648773193 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.867687463760376 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.8679089546203613 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.8678581714630127 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.8678600788116455 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.8679914474487305 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.8676421642303467 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.866832971572876 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.8673715591430664 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.8678665161132812 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.8678536415100098 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.8678712844848633 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.867645263671875 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.867842197418213 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.8671417236328125 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.8673973083496094 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.8678674697875977 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.8678786754608154 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.867872714996338 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.8679234981536865 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.8680260181427 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.8681395053863525 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.867269992828369 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.867920160293579 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.8678531646728516 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.8679559230804443 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.8679633140563965 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.867887020111084 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.8676884174346924 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.8678176403045654 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.867934226989746 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.867680788040161 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.867964267730713 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.8679592609405518 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.867408275604248 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.867422103881836 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.8674020767211914 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.8679192066192627 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.8677475452423096 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.8680362701416016 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.8676185607910156 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.8673884868621826 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.8676412105560303 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.8676092624664307 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.867985963821411 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.867816686630249 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.8630337715148926 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.867835283279419 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.867220640182495 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.86899733543396 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.8678879737854004 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.8678665161132812 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.8679568767547607 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.8665947914123535 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.8676748275756836 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.8672592639923096 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.868016004562378 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.8680524826049805 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.8677475452423096 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.8681135177612305 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.867318868637085 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.8671715259552 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.867291212081909 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.8673880100250244 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.8679707050323486 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.8682870864868164 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.868133783340454 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.865920066833496 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.867737054824829 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.86694598197937 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.867063522338867 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.867006301879883 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.8691718578338623 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.8720345497131348 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.8621394634246826 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.8672947883605957 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.8673691749572754 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.867260694503784 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.8689162731170654 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.8677828311920166 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.8697781562805176 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.867631196975708 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.8670198917388916 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.8679425716400146 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.868257999420166 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.8663437366485596 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.8677453994750977 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.8683414459228516 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.866899251937866 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.867821455001831 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.8674843311309814 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.8680994510650635 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.86787486076355 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.8678090572357178 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.8679089546203613 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.8677237033843994 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.867786407470703 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.8673574924468994 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.8682141304016113 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.8675761222839355 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.868237257003784 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.868049383163452 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.8679275512695312 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.8679275512695312 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.8666815757751465 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.8669333457946777 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.865738868713379 + } + ] + }, + { + "bit": 12, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.798635959625244 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.798552989959717 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.7984533309936523 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.7985141277313232 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.7986092567443848 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.7985315322875977 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.7985575199127197 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.798753023147583 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.799036741256714 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.798588275909424 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.7990286350250244 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.798705816268921 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.798692464828491 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.7987539768218994 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.7988147735595703 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7986183166503906 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.798799514770508 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.798739194869995 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.79856538772583 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.7988386154174805 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.798699378967285 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.798915386199951 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.798729419708252 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7987866401672363 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.7984344959259033 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.798835515975952 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.798701524734497 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.798571825027466 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7987375259399414 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.7984416484832764 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.798694372177124 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.7987313270568848 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.7984931468963623 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.7983500957489014 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.7984089851379395 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.798785448074341 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.7987101078033447 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.7987141609191895 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.7986412048339844 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.798553943634033 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.798563003540039 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.798513174057007 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.798788547515869 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.798562526702881 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.7983524799346924 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.798398017883301 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.798713445663452 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.7986299991607666 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.798640012741089 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.7984914779663086 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7986199855804443 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7984654903411865 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.798764944076538 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.798654794692993 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.7985360622406006 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.7983148097991943 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.798701524734497 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.798574686050415 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.7986626625061035 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7985951900482178 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7984914779663086 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.7988791465759277 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.7984397411346436 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.79860258102417 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.7986974716186523 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.7987866401672363 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.7985239028930664 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.7984721660614014 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.7986035346984863 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.798732280731201 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.7987313270568848 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.7986021041870117 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.798682451248169 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.7986114025115967 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.798701763153076 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7989392280578613 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.7983148097991943 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.798732280731201 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.7988035678863525 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.7986254692077637 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.798642158508301 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.7986299991607666 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.7985050678253174 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.7986042499542236 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.798642635345459 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.7987124919891357 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.7989625930786133 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.7987093925476074 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7986462116241455 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.7986667156219482 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.7987477779388428 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.7985827922821045 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.798373222351074 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.798797607421875 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.798219680786133 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.798762083053589 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.798408031463623 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.7983834743499756 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7984766960144043 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7981984615325928 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.7987453937530518 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.798570394515991 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.798661708831787 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.798668622970581 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7987749576568604 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.7985951900482178 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7988102436065674 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.7983908653259277 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.79829478263855 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.798801898956299 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.7987611293792725 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.7985727787017822 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.798748254776001 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.7987749576568604 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7982306480407715 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.7986154556274414 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.7984766960144043 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.7984910011291504 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.7986834049224854 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.798642635345459 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.798381805419922 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.798490047454834 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.7985856533050537 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.7984604835510254 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7985429763793945 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.798581600189209 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.7983882427215576 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7986953258514404 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.7984354496002197 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.7984633445739746 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.798670768737793 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7985892295837402 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.798487424850464 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.7986536026000977 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.79880428314209 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.798362970352173 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7987513542175293 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.7986843585968018 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.79882550239563 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.798262357711792 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.798811197280884 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.7987220287323 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.79838228225708 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.7985827922821045 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.798670768737793 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7985150814056396 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.7988529205322266 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.7987895011901855 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7986655235290527 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.798051118850708 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.798435688018799 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.7982771396636963 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.798436403274536 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.798671007156372 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.7983927726745605 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.798685312271118 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.797586441040039 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.798736333847046 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.798774003982544 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.7985992431640625 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.798456907272339 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.7984859943389893 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.7985947132110596 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.7980079650878906 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.798506021499634 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.7991676330566406 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.7983744144439697 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.7982559204101562 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.798624038696289 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.798701524734497 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7987172603607178 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.7981972694396973 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.798795700073242 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.7988393306732178 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.7989001274108887 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.7984869480133057 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.7984886169433594 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.7988102436065674 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.7985520362854004 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.7987351417541504 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.7984941005706787 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.7982404232025146 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.798790454864502 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.7987172603607178 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.7983028888702393 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.798712730407715 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.798628091812134 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.798147201538086 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.798633098602295 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.7985072135925293 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.7987468242645264 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7965102195739746 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.798616409301758 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.798619031906128 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7988038063049316 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.798548460006714 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.798565626144409 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.798806667327881 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7989695072174072 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.798596143722534 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.7982177734375 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.7983148097991943 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.7985429763793945 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.7984824180603027 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.7987213134765625 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.798776626586914 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.7989180088043213 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.7984020709991455 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.7982821464538574 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.798452615737915 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.7986462116241455 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.798631191253662 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7989377975463867 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.7987289428710938 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.798544406890869 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.79844069480896 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7983005046844482 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.798443555831909 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.7986485958099365 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.798156261444092 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.798711061477661 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.7988529205322266 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.7985329627990723 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.798234224319458 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.7984354496002197 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.798691511154175 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.7984256744384766 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.797917127609253 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.7981646060943604 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.7989253997802734 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.798417091369629 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7986807823181152 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.7987427711486816 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.7983057498931885 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.7990729808807373 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.798542022705078 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.7989397048950195 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.7989838123321533 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.79892635345459 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.798701524734497 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.7992937564849854 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.798884630203247 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7983028888702393 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.798220634460449 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.7990126609802246 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.798788547515869 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7983932495117188 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7982630729675293 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.7985262870788574 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.79862380027771 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.798283100128174 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.7986371517181396 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.798344612121582 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.7988481521606445 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.798112154006958 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.7987260818481445 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.7986409664154053 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.7985665798187256 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.799027681350708 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.7987053394317627 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7985727787017822 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.7985970973968506 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.798114538192749 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.7988336086273193 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.798694372177124 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.7979652881622314 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.79858660697937 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.7984325885772705 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.7983334064483643 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.798727035522461 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.7983884811401367 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.798387289047241 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.7987568378448486 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.7985012531280518 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.7986578941345215 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.798341751098633 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.798879861831665 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.798595428466797 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.798715114593506 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.798478603363037 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.798672676086426 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.798506021499634 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.799117088317871 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.798715114593506 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.798236131668091 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.798455238342285 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.7987730503082275 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.798788547515869 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.798784017562866 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.7986209392547607 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.798525333404541 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.7985565662384033 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.7985012531280518 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.798574686050415 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.7988743782043457 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.79860258102417 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.798544406890869 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.7986369132995605 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.798524856567383 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.798711061477661 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.7982306480407715 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7987334728240967 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7988836765289307 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.7985451221466064 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.798816442489624 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.798701286315918 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7986557483673096 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.7983953952789307 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.7986462116241455 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.798694372177124 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.798614501953125 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.798598051071167 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.7982778549194336 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.798718214035034 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.798959493637085 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.798706531524658 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.798640012741089 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.798619031906128 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.798571825027466 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.798661708831787 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.7982125282287598 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.7985050678253174 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.7987539768218994 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.798691511154175 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7986104488372803 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.7986817359924316 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.7985787391662598 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.798194408416748 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.7988133430480957 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.7985873222351074 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.7986977100372314 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.7987537384033203 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.7986834049224854 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7983946800231934 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.798903465270996 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.7980332374572754 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.7988386154174805 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.798672914505005 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.798764944076538 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.79841685295105 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.7985377311706543 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.7985634803771973 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.7985525131225586 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.798619508743286 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.7986361980438232 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.7987303733825684 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.798727035522461 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.7985594272613525 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7981860637664795 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7984459400177 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.7986092567443848 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.7987568378448486 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.7986185550689697 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7983882427215576 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.798020362854004 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.7987570762634277 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.798306941986084 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.798720598220825 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.798830270767212 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7985050678253174 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.798762798309326 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.798462390899658 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.798220157623291 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.7985923290252686 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7985920906066895 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7988481521606445 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.798642635345459 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.79860520362854 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.7982523441314697 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.79813814163208 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.7982921600341797 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7987802028656006 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.7987847328186035 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.798657178878784 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.798593759536743 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.7983927726745605 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.7985012531280518 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.7984488010406494 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.7987587451934814 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.798621416091919 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.7986741065979004 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.798888921737671 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.798637866973877 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.7985150814056396 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.7988879680633545 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.798692464828491 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.798487901687622 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7989513874053955 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.7984256744384766 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.798053026199341 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.79874849319458 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.7985289096832275 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.798746347427368 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.79854679107666 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.798679828643799 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.798632860183716 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.79887318611145 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.7985622882843018 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.799034833908081 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7985992431640625 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.798725128173828 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.798548936843872 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.7986648082733154 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.798679828643799 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.797783136367798 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.798424005508423 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.798762083053589 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.798487901687622 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.798666000366211 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.79862904548645 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.7988295555114746 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.798600196838379 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.7987618446350098 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.798715114593506 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.798598051071167 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.798705816268921 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.798698663711548 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.798501968383789 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.7986338138580322 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.7983949184417725 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.798718214035034 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.7985801696777344 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.7986531257629395 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.798490524291992 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.7986466884613037 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.798679828643799 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.7980568408966064 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.798529624938965 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.7986984252929688 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.7986035346984863 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.798598051071167 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.7986695766448975 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.7982072830200195 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.798248529434204 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.798624038696289 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.7988197803497314 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.7986254692077637 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7985305786132812 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.7978312969207764 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7983803749084473 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7984941005706787 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.7985613346099854 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.7987701892852783 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.7984960079193115 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.798567771911621 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.7984931468963623 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.7984445095062256 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.7981202602386475 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.798571825027466 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.798591136932373 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7989840507507324 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.7985222339630127 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.798511266708374 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7981021404266357 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.798596143722534 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.798715591430664 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.7987701892852783 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.7986719608306885 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.798449993133545 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7984695434570312 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.7988383769989014 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.798410177230835 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.7986044883728027 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.7985315322875977 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.7983484268188477 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.7988317012786865 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.7982568740844727 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.7984588146209717 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.798675060272217 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.798661708831787 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7986741065979004 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.7986321449279785 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.798583745956421 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.798041582107544 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.7980306148529053 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.7987568378448486 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.7985637187957764 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.798708438873291 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.7985126972198486 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.798642635345459 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.79849910736084 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.798433542251587 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.798006057739258 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.7987678050994873 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.7986154556274414 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.7980458736419678 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.7986044883728027 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.79882550239563 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.798675060272217 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.798778533935547 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.798574447631836 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.79874849319458 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.798283576965332 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.7982430458068848 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.798159122467041 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7989773750305176 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.798170804977417 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.798570394515991 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.7985994815826416 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.7920145988464355 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.7985458374023438 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.797701835632324 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.799617290496826 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.798402786254883 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.7985897064208984 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.798567533493042 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.79725980758667 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.7983007431030273 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.797891855239868 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7980477809906006 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.797978401184082 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.798570394515991 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.7988672256469727 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.7973668575286865 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.798506259918213 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.798513174057007 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.7986717224121094 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.7989108562469482 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.798548460006714 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.7988555431365967 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.797884225845337 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.7986669540405273 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.7972872257232666 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7997405529022217 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7983527183532715 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7990286350250244 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.797955274581909 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.7931339740753174 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.7987003326416016 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.7980570793151855 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7992732524871826 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.7973830699920654 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.7989377975463867 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7992966175079346 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.7977261543273926 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.7985458374023438 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.798105001449585 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.7988386154174805 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.7994933128356934 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.798638105392456 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.798480272293091 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.7975564002990723 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.798473358154297 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.7989513874053955 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.798661708831787 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.7978515625 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.7989296913146973 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.798449754714966 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.7981786727905273 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.79840087890625 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.7981889247894287 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.7990493774414062 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.798431634902954 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.798750400543213 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.7989702224731445 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.798811197280884 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.798750400543213 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7984695434570312 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.7982311248779297 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.797874927520752 + } + ] + }, + { + "bit": 13, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.738034248352051 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.7379770278930664 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.738163709640503 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.738264560699463 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.7380359172821045 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.738380193710327 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.7382194995880127 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.7383220195770264 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.738025188446045 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.73818302154541 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.738260507583618 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.7379446029663086 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.7380752563476562 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.737929105758667 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.7381105422973633 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.7382266521453857 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.738213300704956 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.7380948066711426 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.738168954849243 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.7383153438568115 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.7378828525543213 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.7380313873291016 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.737989664077759 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.7380740642547607 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.7379212379455566 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.738199472427368 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.737938404083252 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.73827862739563 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.7380948066711426 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.7379798889160156 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.738118886947632 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.7381746768951416 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.7381579875946045 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.738070249557495 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.7381556034088135 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.738213300704956 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.738260507583618 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.737927198410034 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.738363265991211 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.738253116607666 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.7380876541137695 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.7380661964416504 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.738093852996826 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.738032102584839 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.7382891178131104 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.7381339073181152 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.7383248805999756 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.737891435623169 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.738313674926758 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.7381670475006104 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.7382185459136963 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.7382423877716064 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.7382302284240723 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.7382161617279053 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.7378246784210205 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.737987518310547 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.7381296157836914 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.738034725189209 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.738194227218628 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.7379844188690186 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.7382071018218994 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.7378556728363037 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.738112688064575 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.7380149364471436 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.738203525543213 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.7381515502929688 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.738220453262329 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.7382354736328125 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.738161087036133 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.738081216812134 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.7382614612579346 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.738050937652588 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.7380530834198 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.738232135772705 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.7383220195770264 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.7382302284240723 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.738266706466675 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.738151788711548 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.7380261421203613 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.738051176071167 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.7381465435028076 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.738206148147583 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.738239288330078 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.7379746437072754 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.738337278366089 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.738151788711548 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.7381386756896973 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.738233804702759 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.7381043434143066 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.7382009029388428 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.7379636764526367 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.7382054328918457 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.7380993366241455 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.7379915714263916 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.7382874488830566 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.738384485244751 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.7381105422973633 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.7383131980895996 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.7382113933563232 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.7383081912994385 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.737860918045044 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.7378275394439697 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.738308906555176 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.738065481185913 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.7383737564086914 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.73837947845459 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.7381937503814697 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.738292932510376 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.738067150115967 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.738471031188965 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.738046884536743 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.7383434772491455 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.738233804702759 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.7382614612579346 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.7382736206054688 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.738035202026367 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.738068103790283 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.738065004348755 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.7381162643432617 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.7382779121398926 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.7382609844207764 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.7379958629608154 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.7381651401519775 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.7383689880371094 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.7381680011749268 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.7380239963531494 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.7381410598754883 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.7380659580230713 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.737990617752075 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.738281011581421 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.7381386756896973 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.7382354736328125 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.7380282878875732 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.7381036281585693 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.738279342651367 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.7378408908843994 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.7378058433532715 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.737926483154297 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.7380714416503906 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.7384073734283447 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.738408327102661 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.738250732421875 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.7379636764526367 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.737947463989258 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.7382779121398926 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.7380030155181885 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.738126039505005 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.738405704498291 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.7379024028778076 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.738192081451416 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.738326072692871 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.7380430698394775 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.738435983657837 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.737940549850464 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.738077163696289 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.738102674484253 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.7391583919525146 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.738118886947632 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.7381324768066406 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.7381482124328613 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.7380948066711426 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.7382802963256836 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.738332986831665 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.7380459308624268 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.738126039505005 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.737997055053711 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.7383530139923096 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.738346576690674 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.7381722927093506 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.7383604049682617 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.7382373809814453 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.7381701469421387 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.738049268722534 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.737795352935791 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.738077163696289 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.738189220428467 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.738271474838257 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.737966537475586 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.7380740642547607 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.7382724285125732 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.7380189895629883 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.738273859024048 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.7382473945617676 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.738095283508301 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.73834490776062 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.737762451171875 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.73816180229187 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.7382864952087402 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.738029956817627 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.7382123470306396 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.7381577491760254 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.7380106449127197 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.7379708290100098 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.7384557723999023 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.7376046180725098 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.738595724105835 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.7381999492645264 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.7380170822143555 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.7375600337982178 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.7379300594329834 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.738407611846924 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.73865008354187 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.7384347915649414 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.738262414932251 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.73819637298584 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.7377676963806152 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.737790107727051 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.738163709640503 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.738147735595703 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.7380378246307373 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.7380661964416504 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.738126039505005 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.7370243072509766 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.738306999206543 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.7381999492645264 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.737987756729126 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.7380428314208984 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.738213300704956 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.738054037094116 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.738208293914795 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.738168954849243 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.738206624984741 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.7385034561157227 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.7383880615234375 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.7384047508239746 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.738342523574829 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.738750457763672 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.738602638244629 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.738337278366089 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.7375590801239014 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.7378456592559814 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.7385900020599365 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.738111734390259 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.7383158206939697 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.7379870414733887 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.738417625427246 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.738027811050415 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.738264322280884 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.7382819652557373 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.738450527191162 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.737630605697632 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.7380118370056152 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.7379255294799805 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.7381017208099365 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.7380411624908447 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.7382655143737793 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.7381913661956787 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.7379164695739746 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.738042116165161 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.73795223236084 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.7386553287506104 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.73776912689209 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.7380974292755127 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.7382302284240723 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.7379891872406006 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.737891435623169 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.738476276397705 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.738297700881958 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.737607479095459 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.7384281158447266 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.7382538318634033 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.737872838973999 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.7385573387145996 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.737908124923706 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.7381157875061035 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.73858380317688 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.7383034229278564 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.738253593444824 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.7385172843933105 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.7381515502929688 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.738163948059082 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.7381045818328857 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.7376012802124023 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.73830509185791 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.738265037536621 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.738119602203369 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.7382233142852783 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.7376933097839355 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.7380764484405518 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.7378087043762207 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.7381083965301514 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.7383835315704346 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.738039493560791 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.7384042739868164 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.7385785579681396 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.7384538650512695 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.737679958343506 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.738149642944336 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.738185167312622 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.738356351852417 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.7382194995880127 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.7378721237182617 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.7380781173706055 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.7378668785095215 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.73822021484375 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.7381839752197266 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.7384297847747803 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.7382538318634033 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.7379322052001953 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.7376608848571777 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.738239288330078 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.7383873462677 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.7382619380950928 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.7384397983551025 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.7381224632263184 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.7380623817443848 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.7379798889160156 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.7379939556121826 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.7381999492645264 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.738267660140991 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.7382123470306396 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.7380549907684326 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.738093852996826 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.737955331802368 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.7374937534332275 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.7382726669311523 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.738454580307007 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.7383010387420654 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.7382214069366455 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.7379915714263916 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.7380685806274414 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.73795747756958 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.738257884979248 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.738236427307129 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.7383270263671875 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.738058567047119 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.737872362136841 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.738034248352051 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.7380428314208984 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.738347053527832 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.738222599029541 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.738203525543213 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.7382521629333496 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.7381722927093506 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.7375168800354004 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.738079309463501 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.738107681274414 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.738271474838257 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.738112688064575 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.7381491661071777 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.738105535507202 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.7380459308624268 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.7378053665161133 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.7382946014404297 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.738274574279785 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.73823618888855 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.73827862739563 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.7380850315093994 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.7385754585266113 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.7381696701049805 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.7382233142852783 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.7382071018218994 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.738260507583618 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.7382864952087402 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.7386553287506104 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.7378921508789062 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.7377824783325195 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.738301992416382 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.7382562160491943 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.7381417751312256 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.7384047508239746 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.7378268241882324 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.7379324436187744 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.7377383708953857 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.7382802963256836 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.7384538650512695 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.7380623817443848 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.7382311820983887 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.738070249557495 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.7380473613739014 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.737870693206787 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.7384228706359863 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.738250732421875 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.7381982803344727 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.7380921840667725 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.738145589828491 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.738220453262329 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.737788438796997 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.7381439208984375 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.7382044792175293 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.738072156906128 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.738243579864502 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.7377943992614746 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.7379424571990967 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.7377278804779053 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.7382960319519043 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.738267660140991 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.7383220195770264 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.738252639770508 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.738414764404297 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.737912178039551 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.737980842590332 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.7381982803344727 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.738339424133301 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.738171100616455 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.7379705905914307 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.7380123138427734 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.737823486328125 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.7379307746887207 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.7382960319519043 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.7381701469421387 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.7384626865386963 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.738192319869995 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.7379815578460693 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.7383241653442383 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.737922430038452 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.738250732421875 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.738208055496216 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.738363265991211 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.738208293914795 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.7379746437072754 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.737844467163086 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.7380459308624268 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.7381224632263184 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.7382988929748535 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.7381973266601562 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.7382326126098633 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.7382102012634277 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.738011121749878 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.7379939556121826 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.7382373809814453 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.7381298542022705 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.738358497619629 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.738474130630493 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.7380733489990234 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.737700939178467 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.738553762435913 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.738098621368408 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.7382311820983887 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.7382946014404297 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.7383193969726562 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.7383363246917725 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.738297939300537 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.738187074661255 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.738314151763916 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.7382242679595947 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.7385075092315674 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.7383577823638916 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.738429546356201 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.7377564907073975 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.7381722927093506 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.7383127212524414 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.73824143409729 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.7382545471191406 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.7382614612579346 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.737879753112793 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.7381722927093506 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.73828387260437 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.7383008003234863 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.738288164138794 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.7379424571990967 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.7383382320404053 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.7383623123168945 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.7384514808654785 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.738121509552002 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.7383158206939697 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.738269567489624 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.738530158996582 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.7384064197540283 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.7379343509674072 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.7375950813293457 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.7377922534942627 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.7382140159606934 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.7381932735443115 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.738370180130005 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.738237142562866 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.7380306720733643 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.737889051437378 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.738114833831787 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.7383105754852295 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.7382071018218994 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.7384276390075684 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.7382290363311768 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.73842453956604 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.7383060455322266 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.7381036281585693 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.7381763458251953 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.7382242679595947 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.7381439208984375 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.7382171154022217 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.7383148670196533 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.7377374172210693 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.7383038997650146 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.73832106590271 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.738215446472168 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.7376039028167725 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.738269805908203 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.737642288208008 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.738002061843872 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.737542152404785 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.738253116607666 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.7380762100219727 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.7381205558776855 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.73807430267334 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.7380740642547607 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.7378146648406982 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.73775315284729 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.738375425338745 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.738173246383667 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.7292778491973877 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.738227605819702 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.737367630004883 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.738783836364746 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.738006591796875 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.7381937503814697 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.7381982803344727 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.73649263381958 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.738184928894043 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.7379093170166016 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.7382583618164062 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.738426446914673 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.738128423690796 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.738342523574829 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.7374038696289062 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.737607717514038 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.737753391265869 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.7385432720184326 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.738204002380371 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.738255739212036 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.7384157180786133 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.7372775077819824 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.7381861209869385 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.738107442855835 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.7375428676605225 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.7375943660736084 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.7391295433044434 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.7418136596679688 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.7302491664886475 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.7377514839172363 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.7378971576690674 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.7379512786865234 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.7390708923339844 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.73827862739563 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.7391576766967773 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.7384700775146484 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.737765073776245 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.738361358642578 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.7382707595825195 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.7385311126708984 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.7382333278656006 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.7385787963867188 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.736440658569336 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.7381420135498047 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.737760305404663 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.7384419441223145 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.738093852996826 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.738109827041626 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.738190174102783 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.7381539344787598 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.7383382320404053 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.7379112243652344 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.738466739654541 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.7387688159942627 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.7385098934173584 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.738359212875366 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.7381649017333984 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.7382583618164062 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.7375762462615967 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.737764596939087 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.736659049987793 + } + ] + }, + { + "bit": 14, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.681103229522705 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.68114972114563 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.681140661239624 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.6810216903686523 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.6810386180877686 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.680908441543579 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.6809027194976807 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.680833339691162 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.680882692337036 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.6810057163238525 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.6810543537139893 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.6811535358428955 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.6810624599456787 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.6809325218200684 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.6808464527130127 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.681044340133667 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.681166172027588 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.681053638458252 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.6810364723205566 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.6809451580047607 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.680936574935913 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.680981159210205 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.680896520614624 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.6809520721435547 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.6808905601501465 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.681008815765381 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.6811726093292236 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.680900812149048 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.6809237003326416 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.680952310562134 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.6808440685272217 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.681087017059326 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.681056261062622 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.6809451580047607 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.681062936782837 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.6810615062713623 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.6810150146484375 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.6808629035949707 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.681195020675659 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.681022882461548 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.6809167861938477 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.6809966564178467 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.680945634841919 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.6811721324920654 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.6809213161468506 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.6807620525360107 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.6810367107391357 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.680854320526123 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.680795907974243 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.6809163093566895 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.680893898010254 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.6811347007751465 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.6810784339904785 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.6810755729675293 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.6810123920440674 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.6809444427490234 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.6810781955718994 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.6810035705566406 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.681030750274658 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.681138753890991 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.680985689163208 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.680964469909668 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.681053638458252 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.6810431480407715 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.6809494495391846 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.6808650493621826 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.680854320526123 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.680785894393921 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.680954933166504 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.681016445159912 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.681074619293213 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.6810808181762695 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.680948257446289 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.681224822998047 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.6809206008911133 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.680704355239868 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.681180000305176 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.681049346923828 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.6809284687042236 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.6810553073883057 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.68118953704834 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.6809659004211426 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.6809635162353516 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.6812446117401123 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.68106746673584 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.68101167678833 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.6810967922210693 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.680922508239746 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.680931329727173 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.6810503005981445 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.6810035705566406 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.6811275482177734 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.6810266971588135 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.6811060905456543 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.6812050342559814 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.681053638458252 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.680873155593872 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.681198835372925 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.680910587310791 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.6808571815490723 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.680980920791626 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.6810755729675293 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.6809561252593994 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.6810097694396973 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.6811277866363525 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.6808977127075195 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.6810054779052734 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.6810288429260254 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.6807937622070312 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.680806875228882 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.681081771850586 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.6808385848999023 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.681074619293213 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.6810245513916016 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.6812329292297363 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.681013822555542 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.681304454803467 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.6810495853424072 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.6812257766723633 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.6810414791107178 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.6808273792266846 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.681062936782837 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.6808078289031982 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.6808974742889404 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.6810288429260254 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.6810591220855713 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.6810991764068604 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.6809422969818115 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.6808109283447266 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.6807868480682373 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.681079387664795 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.6810057163238525 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.681124687194824 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.68109130859375 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.681101083755493 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.6808650493621826 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.6811330318450928 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.680995225906372 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.6811423301696777 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.6806912422180176 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.6812031269073486 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.681201934814453 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.681102752685547 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.6810035705566406 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.6809394359588623 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.6809821128845215 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.6809771060943604 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.681013345718384 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.6808762550354004 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.6802008152008057 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.6810781955718994 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.6810123920440674 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.681091785430908 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.68111252784729 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.680989980697632 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.6808369159698486 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.6792309284210205 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.6809535026550293 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.681278705596924 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.6811115741729736 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.6809747219085693 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.6809921264648438 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.6808881759643555 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.680663585662842 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.6807913780212402 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.6811563968658447 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.6806797981262207 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.680750846862793 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.6809699535369873 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.681044340133667 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.680678606033325 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.680809259414673 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.6811535358428955 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.6808176040649414 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.6811304092407227 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.6809325218200684 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.6811060905456543 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.6810295581817627 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.680936098098755 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.6809325218200684 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.6811752319335938 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.680670976638794 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.68091082572937 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.680864095687866 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.6805758476257324 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.6810660362243652 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.6809990406036377 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.6806085109710693 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.680999994277954 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.680734157562256 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.6812071800231934 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.6783127784729004 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.680737018585205 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.6808841228485107 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.6812922954559326 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.680680990219116 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.681087017059326 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.681180953979492 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.681703805923462 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.6815850734710693 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.680727481842041 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.680621862411499 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.6805644035339355 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.68111252784729 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.6811230182647705 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.6814486980438232 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.6813485622406006 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.680899143218994 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.6805331707000732 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.68082332611084 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.6807332038879395 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.6810719966888428 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.681809902191162 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.6811132431030273 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.6811256408691406 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.6808292865753174 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.6808788776397705 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.681093215942383 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.680929183959961 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.6807737350463867 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.68086314201355 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.681114912033081 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.680874824523926 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.6810801029205322 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.6809017658233643 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.6808860301971436 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.680528402328491 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.6806769371032715 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.6807727813720703 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.681241512298584 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.6810505390167236 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.680866003036499 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.6809241771698 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.681032180786133 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.681058168411255 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.681028366088867 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.681101083755493 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.6809604167938232 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.681145429611206 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.680972099304199 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.6807165145874023 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.680962324142456 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.680771589279175 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.6809604167938232 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.6811177730560303 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.681001901626587 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.6809446811676025 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.68086838722229 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.681257724761963 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.6810684204101562 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.680879831314087 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.681137800216675 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.6809747219085693 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.681020498275757 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.680743932723999 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.6810595989227295 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.680849075317383 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.680845260620117 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.680921792984009 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.680849075317383 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.680891752243042 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.6812307834625244 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.680445909500122 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.6812031269073486 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.6807265281677246 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.6806912422180176 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.6811633110046387 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.6811177730560303 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.680771589279175 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.6808693408966064 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.681267499923706 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.681035280227661 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.681124448776245 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.6809821128845215 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.6811444759368896 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.6807727813720703 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.6810333728790283 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.681072950363159 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.6811060905456543 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.6808342933654785 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.6810238361358643 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.6809237003326416 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.6810834407806396 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.681051015853882 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.680596351623535 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.6807384490966797 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.681197166442871 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.6811914443969727 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.6810574531555176 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.6809780597686768 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.6807072162628174 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.680490493774414 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.6810503005981445 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.681074619293213 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.6810061931610107 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.6810195446014404 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.681016445159912 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.681060552597046 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.681191921234131 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.6808502674102783 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.6809372901916504 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.6809377670288086 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.680950164794922 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.6809070110321045 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.6812281608581543 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.6811535358428955 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.680962324142456 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.6809699535369873 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.681061267852783 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.6811091899871826 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.681025505065918 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.680960178375244 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.681022882461548 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.6807172298431396 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.680877208709717 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.6810572147369385 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.6809966564178467 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.6811556816101074 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.6810061931610107 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.6809682846069336 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.6805520057678223 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.680990219116211 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.681015729904175 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.6812515258789062 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.6811435222625732 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.681001663208008 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.681318759918213 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.681040048599243 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.681058168411255 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.6811888217926025 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.681105375289917 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.6812117099761963 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.680997848510742 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.6809957027435303 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.681180000305176 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.680891990661621 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.6812119483947754 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.681156635284424 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.681074619293213 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.681025981903076 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.681220769882202 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.6808242797851562 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.6812007427215576 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.6809918880462646 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.681131362915039 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.681217908859253 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.6811037063598633 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.680647134780884 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.680631399154663 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.680938243865967 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.6809659004211426 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.681187391281128 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.6808762550354004 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.680973768234253 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.680454969406128 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.6811554431915283 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.681150197982788 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.6810526847839355 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.6811141967773438 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.681147575378418 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.681095838546753 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.6811439990997314 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.680859088897705 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.6811506748199463 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.681060552597046 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.6812355518341064 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.6810145378112793 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.681213855743408 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.6810851097106934 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.6808323860168457 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.6807878017425537 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.681104898452759 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.6810691356658936 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.681087017059326 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.6809561252593994 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.680997848510742 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.680649995803833 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.6811277866363525 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.6809897422790527 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.6809909343719482 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.681044340133667 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.6810171604156494 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.6810615062713623 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.6810028553009033 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.6807878017425537 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.6811940670013428 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.6810615062713623 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.680996894836426 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.680908679962158 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.680835485458374 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.68110990524292 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.680752754211426 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.681044816970825 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.680979013442993 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.6811676025390625 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.6811177730560303 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.681224822998047 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.681209087371826 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.681147813796997 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.6810977458953857 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.681058168411255 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.6810505390167236 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.6809921264648438 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.6809494495391846 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.6807847023010254 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.6811745166778564 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.681051731109619 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.6809980869293213 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.680900812149048 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.6809325218200684 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.6811466217041016 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.681251049041748 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.681049346923828 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.681058168411255 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.6810054779052734 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.6810078620910645 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.6811325550079346 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.6807210445404053 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.68088960647583 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.6812634468078613 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.6810503005981445 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.6808574199676514 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.6809394359588623 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.680922746658325 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.681039571762085 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.6811070442199707 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.6807281970977783 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.680999994277954 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.6810543537139893 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.681070327758789 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.681004524230957 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.680842161178589 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.6808464527130127 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.680697441101074 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.680885076522827 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.681060314178467 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.6808881759643555 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.681095838546753 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.680704355239868 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.681137800216675 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.6808176040649414 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.681014060974121 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.681046485900879 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.6809911727905273 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.681126594543457 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.6810505390167236 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.6808698177337646 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.680906057357788 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.6810145378112793 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.6810660362243652 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.6811740398406982 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.680907964706421 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.6808342933654785 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.6806998252868652 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.68091082572937 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.6809897422790527 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.681046485900879 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.6810238361358643 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.680809736251831 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.681124687194824 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.681353807449341 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.6810896396636963 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.681041717529297 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.681016683578491 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.6810009479522705 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.680861234664917 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.681119680404663 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.6812028884887695 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.6807491779327393 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.6810808181762695 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.680980920791626 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.681006908416748 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.680919647216797 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.6805481910705566 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.680763006210327 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.680785894393921 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.6809659004211426 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.6811633110046387 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.6808888912200928 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.680962324142456 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.6807572841644287 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.6811683177948 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.6806015968322754 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.680929183959961 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.6810407638549805 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.680710792541504 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.681053638458252 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.681145429611206 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.6809139251708984 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.6811115741729736 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.6810317039489746 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.680948257446289 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.679795742034912 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.6809256076812744 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.6808395385742188 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.6811344623565674 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.680572748184204 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.6809709072113037 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.680989980697632 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.6676764488220215 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.6809465885162354 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.6805903911590576 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.68176531791687 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.6808104515075684 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.681149482727051 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.681072950363159 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.6790611743927 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.680809259414673 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.680624485015869 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.6811413764953613 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.680722236633301 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.6811070442199707 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.6814606189727783 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.679833173751831 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.681013345718384 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.680664539337158 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.6807780265808105 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.6809966564178467 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.6811423301696777 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.6811137199401855 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.680504322052002 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.6809771060943604 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.679739236831665 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.6821157932281494 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.6807913780212402 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.681607484817505 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.6800355911254883 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.668833017349243 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.681086301803589 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.680502414703369 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.681919813156128 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.6805331707000732 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.68142032623291 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.6818580627441406 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.679971694946289 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.6812191009521484 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.680605173110962 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.6813926696777344 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.6817445755004883 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.681006908416748 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.6810591220855713 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.680298089981079 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.6808016300201416 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.680999994277954 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.6814675331115723 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.681518077850342 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.681196928024292 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.680927276611328 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.680706262588501 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.680946111679077 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.68076491355896 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.6811211109161377 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.6808619499206543 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.6811132431030273 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.6812479496002197 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.6811883449554443 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.6810903549194336 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.680696487426758 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.6806726455688477 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.6799304485321045 + } + ] + }, + { + "bit": 15, + "ppls": [ + { + "layer": "layer.0.self_attn.q_proj", + "ppl": 3.619953155517578 + }, + { + "layer": "layer.0.self_attn.k_proj", + "ppl": 3.620087146759033 + }, + { + "layer": "layer.0.self_attn.v_proj", + "ppl": 3.62021803855896 + }, + { + "layer": "layer.0.self_attn.o_proj", + "ppl": 3.6200413703918457 + }, + { + "layer": "layer.0.mlp.gate_proj", + "ppl": 3.6200058460235596 + }, + { + "layer": "layer.0.mlp.up_proj", + "ppl": 3.6200103759765625 + }, + { + "layer": "layer.0.mlp.down_proj", + "ppl": 3.620183229446411 + }, + { + "layer": "layer.1.self_attn.q_proj", + "ppl": 3.6201443672180176 + }, + { + "layer": "layer.1.self_attn.k_proj", + "ppl": 3.6202871799468994 + }, + { + "layer": "layer.1.self_attn.v_proj", + "ppl": 3.6200144290924072 + }, + { + "layer": "layer.1.self_attn.o_proj", + "ppl": 3.620137929916382 + }, + { + "layer": "layer.1.mlp.gate_proj", + "ppl": 3.620081663131714 + }, + { + "layer": "layer.1.mlp.up_proj", + "ppl": 3.6199636459350586 + }, + { + "layer": "layer.1.mlp.down_proj", + "ppl": 3.62001895904541 + }, + { + "layer": "layer.2.self_attn.q_proj", + "ppl": 3.6201226711273193 + }, + { + "layer": "layer.2.self_attn.k_proj", + "ppl": 3.6200482845306396 + }, + { + "layer": "layer.2.self_attn.v_proj", + "ppl": 3.6202666759490967 + }, + { + "layer": "layer.2.self_attn.o_proj", + "ppl": 3.620089530944824 + }, + { + "layer": "layer.2.mlp.gate_proj", + "ppl": 3.620171546936035 + }, + { + "layer": "layer.2.mlp.up_proj", + "ppl": 3.620208263397217 + }, + { + "layer": "layer.2.mlp.down_proj", + "ppl": 3.6199355125427246 + }, + { + "layer": "layer.3.self_attn.q_proj", + "ppl": 3.6201705932617188 + }, + { + "layer": "layer.3.self_attn.k_proj", + "ppl": 3.6199941635131836 + }, + { + "layer": "layer.3.self_attn.v_proj", + "ppl": 3.620068073272705 + }, + { + "layer": "layer.3.self_attn.o_proj", + "ppl": 3.620049238204956 + }, + { + "layer": "layer.3.mlp.gate_proj", + "ppl": 3.620238780975342 + }, + { + "layer": "layer.3.mlp.up_proj", + "ppl": 3.6200473308563232 + }, + { + "layer": "layer.3.mlp.down_proj", + "ppl": 3.620068073272705 + }, + { + "layer": "layer.4.self_attn.q_proj", + "ppl": 3.620271921157837 + }, + { + "layer": "layer.4.self_attn.k_proj", + "ppl": 3.619934320449829 + }, + { + "layer": "layer.4.self_attn.v_proj", + "ppl": 3.6202573776245117 + }, + { + "layer": "layer.4.self_attn.o_proj", + "ppl": 3.620349884033203 + }, + { + "layer": "layer.4.mlp.gate_proj", + "ppl": 3.6201672554016113 + }, + { + "layer": "layer.4.mlp.up_proj", + "ppl": 3.620192527770996 + }, + { + "layer": "layer.4.mlp.down_proj", + "ppl": 3.6200661659240723 + }, + { + "layer": "layer.5.self_attn.q_proj", + "ppl": 3.620087146759033 + }, + { + "layer": "layer.5.self_attn.k_proj", + "ppl": 3.620069742202759 + }, + { + "layer": "layer.5.self_attn.v_proj", + "ppl": 3.6201331615448 + }, + { + "layer": "layer.5.self_attn.o_proj", + "ppl": 3.62015438079834 + }, + { + "layer": "layer.5.mlp.gate_proj", + "ppl": 3.620074987411499 + }, + { + "layer": "layer.5.mlp.up_proj", + "ppl": 3.6200687885284424 + }, + { + "layer": "layer.5.mlp.down_proj", + "ppl": 3.620084524154663 + }, + { + "layer": "layer.6.self_attn.q_proj", + "ppl": 3.620142698287964 + }, + { + "layer": "layer.6.self_attn.k_proj", + "ppl": 3.620098114013672 + }, + { + "layer": "layer.6.self_attn.v_proj", + "ppl": 3.6200308799743652 + }, + { + "layer": "layer.6.self_attn.o_proj", + "ppl": 3.620222568511963 + }, + { + "layer": "layer.6.mlp.gate_proj", + "ppl": 3.6200268268585205 + }, + { + "layer": "layer.6.mlp.up_proj", + "ppl": 3.6198532581329346 + }, + { + "layer": "layer.6.mlp.down_proj", + "ppl": 3.619924783706665 + }, + { + "layer": "layer.7.self_attn.q_proj", + "ppl": 3.6202590465545654 + }, + { + "layer": "layer.7.self_attn.k_proj", + "ppl": 3.6201157569885254 + }, + { + "layer": "layer.7.self_attn.v_proj", + "ppl": 3.620086193084717 + }, + { + "layer": "layer.7.self_attn.o_proj", + "ppl": 3.6201720237731934 + }, + { + "layer": "layer.7.mlp.gate_proj", + "ppl": 3.6200623512268066 + }, + { + "layer": "layer.7.mlp.up_proj", + "ppl": 3.6202001571655273 + }, + { + "layer": "layer.7.mlp.down_proj", + "ppl": 3.6200404167175293 + }, + { + "layer": "layer.8.self_attn.q_proj", + "ppl": 3.619987726211548 + }, + { + "layer": "layer.8.self_attn.k_proj", + "ppl": 3.619988203048706 + }, + { + "layer": "layer.8.self_attn.v_proj", + "ppl": 3.6200530529022217 + }, + { + "layer": "layer.8.self_attn.o_proj", + "ppl": 3.6202280521392822 + }, + { + "layer": "layer.8.mlp.gate_proj", + "ppl": 3.620256185531616 + }, + { + "layer": "layer.8.mlp.up_proj", + "ppl": 3.620087146759033 + }, + { + "layer": "layer.8.mlp.down_proj", + "ppl": 3.620004177093506 + }, + { + "layer": "layer.9.self_attn.q_proj", + "ppl": 3.6200954914093018 + }, + { + "layer": "layer.9.self_attn.k_proj", + "ppl": 3.62007999420166 + }, + { + "layer": "layer.9.self_attn.v_proj", + "ppl": 3.6202237606048584 + }, + { + "layer": "layer.9.self_attn.o_proj", + "ppl": 3.6202285289764404 + }, + { + "layer": "layer.9.mlp.gate_proj", + "ppl": 3.6203060150146484 + }, + { + "layer": "layer.9.mlp.up_proj", + "ppl": 3.6203510761260986 + }, + { + "layer": "layer.9.mlp.down_proj", + "ppl": 3.620011568069458 + }, + { + "layer": "layer.10.self_attn.q_proj", + "ppl": 3.620079278945923 + }, + { + "layer": "layer.10.self_attn.k_proj", + "ppl": 3.6203293800354004 + }, + { + "layer": "layer.10.self_attn.v_proj", + "ppl": 3.6200153827667236 + }, + { + "layer": "layer.10.self_attn.o_proj", + "ppl": 3.619951009750366 + }, + { + "layer": "layer.10.mlp.gate_proj", + "ppl": 3.6201586723327637 + }, + { + "layer": "layer.10.mlp.up_proj", + "ppl": 3.62007999420166 + }, + { + "layer": "layer.10.mlp.down_proj", + "ppl": 3.6199593544006348 + }, + { + "layer": "layer.11.self_attn.q_proj", + "ppl": 3.619966983795166 + }, + { + "layer": "layer.11.self_attn.k_proj", + "ppl": 3.6200873851776123 + }, + { + "layer": "layer.11.self_attn.v_proj", + "ppl": 3.6199488639831543 + }, + { + "layer": "layer.11.self_attn.o_proj", + "ppl": 3.6200993061065674 + }, + { + "layer": "layer.11.mlp.gate_proj", + "ppl": 3.620155096054077 + }, + { + "layer": "layer.11.mlp.up_proj", + "ppl": 3.6198508739471436 + }, + { + "layer": "layer.11.mlp.down_proj", + "ppl": 3.620126724243164 + }, + { + "layer": "layer.12.self_attn.q_proj", + "ppl": 3.6203255653381348 + }, + { + "layer": "layer.12.self_attn.k_proj", + "ppl": 3.620077610015869 + }, + { + "layer": "layer.12.self_attn.v_proj", + "ppl": 3.620251178741455 + }, + { + "layer": "layer.12.self_attn.o_proj", + "ppl": 3.6199731826782227 + }, + { + "layer": "layer.12.mlp.gate_proj", + "ppl": 3.620164632797241 + }, + { + "layer": "layer.12.mlp.up_proj", + "ppl": 3.6201491355895996 + }, + { + "layer": "layer.12.mlp.down_proj", + "ppl": 3.6201820373535156 + }, + { + "layer": "layer.13.self_attn.q_proj", + "ppl": 3.6200649738311768 + }, + { + "layer": "layer.13.self_attn.k_proj", + "ppl": 3.6201038360595703 + }, + { + "layer": "layer.13.self_attn.v_proj", + "ppl": 3.619978189468384 + }, + { + "layer": "layer.13.self_attn.o_proj", + "ppl": 3.620323419570923 + }, + { + "layer": "layer.13.mlp.gate_proj", + "ppl": 3.6198790073394775 + }, + { + "layer": "layer.13.mlp.up_proj", + "ppl": 3.620152711868286 + }, + { + "layer": "layer.13.mlp.down_proj", + "ppl": 3.620126724243164 + }, + { + "layer": "layer.14.self_attn.q_proj", + "ppl": 3.6202259063720703 + }, + { + "layer": "layer.14.self_attn.k_proj", + "ppl": 3.6199920177459717 + }, + { + "layer": "layer.14.self_attn.v_proj", + "ppl": 3.619856119155884 + }, + { + "layer": "layer.14.self_attn.o_proj", + "ppl": 3.6198203563690186 + }, + { + "layer": "layer.14.mlp.gate_proj", + "ppl": 3.620136260986328 + }, + { + "layer": "layer.14.mlp.up_proj", + "ppl": 3.620236396789551 + }, + { + "layer": "layer.14.mlp.down_proj", + "ppl": 3.620117425918579 + }, + { + "layer": "layer.15.self_attn.q_proj", + "ppl": 3.6202073097229004 + }, + { + "layer": "layer.15.self_attn.k_proj", + "ppl": 3.62015438079834 + }, + { + "layer": "layer.15.self_attn.v_proj", + "ppl": 3.620089530944824 + }, + { + "layer": "layer.15.self_attn.o_proj", + "ppl": 3.619997978210449 + }, + { + "layer": "layer.15.mlp.gate_proj", + "ppl": 3.6200413703918457 + }, + { + "layer": "layer.15.mlp.up_proj", + "ppl": 3.6200344562530518 + }, + { + "layer": "layer.15.mlp.down_proj", + "ppl": 3.620074510574341 + }, + { + "layer": "layer.16.self_attn.q_proj", + "ppl": 3.6201529502868652 + }, + { + "layer": "layer.16.self_attn.k_proj", + "ppl": 3.6201090812683105 + }, + { + "layer": "layer.16.self_attn.v_proj", + "ppl": 3.6198501586914062 + }, + { + "layer": "layer.16.self_attn.o_proj", + "ppl": 3.619929790496826 + }, + { + "layer": "layer.16.mlp.gate_proj", + "ppl": 3.620044708251953 + }, + { + "layer": "layer.16.mlp.up_proj", + "ppl": 3.620072841644287 + }, + { + "layer": "layer.16.mlp.down_proj", + "ppl": 3.6198408603668213 + }, + { + "layer": "layer.17.self_attn.q_proj", + "ppl": 3.620288133621216 + }, + { + "layer": "layer.17.self_attn.k_proj", + "ppl": 3.620091438293457 + }, + { + "layer": "layer.17.self_attn.v_proj", + "ppl": 3.6200335025787354 + }, + { + "layer": "layer.17.self_attn.o_proj", + "ppl": 3.6199228763580322 + }, + { + "layer": "layer.17.mlp.gate_proj", + "ppl": 3.6200034618377686 + }, + { + "layer": "layer.17.mlp.up_proj", + "ppl": 3.620199203491211 + }, + { + "layer": "layer.17.mlp.down_proj", + "ppl": 3.620035171508789 + }, + { + "layer": "layer.18.self_attn.q_proj", + "ppl": 3.6200637817382812 + }, + { + "layer": "layer.18.self_attn.k_proj", + "ppl": 3.620090961456299 + }, + { + "layer": "layer.18.self_attn.v_proj", + "ppl": 3.619882583618164 + }, + { + "layer": "layer.18.self_attn.o_proj", + "ppl": 3.6201012134552 + }, + { + "layer": "layer.18.mlp.gate_proj", + "ppl": 3.6201460361480713 + }, + { + "layer": "layer.18.mlp.up_proj", + "ppl": 3.620300054550171 + }, + { + "layer": "layer.18.mlp.down_proj", + "ppl": 3.6201059818267822 + }, + { + "layer": "layer.19.self_attn.q_proj", + "ppl": 3.620129346847534 + }, + { + "layer": "layer.19.self_attn.k_proj", + "ppl": 3.620161294937134 + }, + { + "layer": "layer.19.self_attn.v_proj", + "ppl": 3.61954665184021 + }, + { + "layer": "layer.19.self_attn.o_proj", + "ppl": 3.620028018951416 + }, + { + "layer": "layer.19.mlp.gate_proj", + "ppl": 3.6197726726531982 + }, + { + "layer": "layer.19.mlp.up_proj", + "ppl": 3.6198172569274902 + }, + { + "layer": "layer.19.mlp.down_proj", + "ppl": 3.620087146759033 + }, + { + "layer": "layer.20.self_attn.q_proj", + "ppl": 3.6201422214508057 + }, + { + "layer": "layer.20.self_attn.k_proj", + "ppl": 3.620142698287964 + }, + { + "layer": "layer.20.self_attn.v_proj", + "ppl": 3.619849443435669 + }, + { + "layer": "layer.20.self_attn.o_proj", + "ppl": 3.619614839553833 + }, + { + "layer": "layer.20.mlp.gate_proj", + "ppl": 3.6198678016662598 + }, + { + "layer": "layer.20.mlp.up_proj", + "ppl": 3.6200368404388428 + }, + { + "layer": "layer.20.mlp.down_proj", + "ppl": 3.6200244426727295 + }, + { + "layer": "layer.21.self_attn.q_proj", + "ppl": 3.620246648788452 + }, + { + "layer": "layer.21.self_attn.k_proj", + "ppl": 3.6198678016662598 + }, + { + "layer": "layer.21.self_attn.v_proj", + "ppl": 3.619946241378784 + }, + { + "layer": "layer.21.self_attn.o_proj", + "ppl": 3.6202335357666016 + }, + { + "layer": "layer.21.mlp.gate_proj", + "ppl": 3.620063304901123 + }, + { + "layer": "layer.21.mlp.up_proj", + "ppl": 3.6198055744171143 + }, + { + "layer": "layer.21.mlp.down_proj", + "ppl": 3.620087146759033 + }, + { + "layer": "layer.22.self_attn.q_proj", + "ppl": 3.6200103759765625 + }, + { + "layer": "layer.22.self_attn.k_proj", + "ppl": 3.620004653930664 + }, + { + "layer": "layer.22.self_attn.v_proj", + "ppl": 3.620281219482422 + }, + { + "layer": "layer.22.self_attn.o_proj", + "ppl": 3.619746208190918 + }, + { + "layer": "layer.22.mlp.gate_proj", + "ppl": 3.620081663131714 + }, + { + "layer": "layer.22.mlp.up_proj", + "ppl": 3.619966983795166 + }, + { + "layer": "layer.22.mlp.down_proj", + "ppl": 3.619802951812744 + }, + { + "layer": "layer.23.self_attn.q_proj", + "ppl": 3.6201283931732178 + }, + { + "layer": "layer.23.self_attn.k_proj", + "ppl": 3.6202173233032227 + }, + { + "layer": "layer.23.self_attn.v_proj", + "ppl": 3.619910955429077 + }, + { + "layer": "layer.23.self_attn.o_proj", + "ppl": 3.6201000213623047 + }, + { + "layer": "layer.23.mlp.gate_proj", + "ppl": 3.6198811531066895 + }, + { + "layer": "layer.23.mlp.up_proj", + "ppl": 3.62025785446167 + }, + { + "layer": "layer.23.mlp.down_proj", + "ppl": 3.620061159133911 + }, + { + "layer": "layer.24.self_attn.q_proj", + "ppl": 3.6200506687164307 + }, + { + "layer": "layer.24.self_attn.k_proj", + "ppl": 3.620217800140381 + }, + { + "layer": "layer.24.self_attn.v_proj", + "ppl": 3.620232582092285 + }, + { + "layer": "layer.24.self_attn.o_proj", + "ppl": 3.6201977729797363 + }, + { + "layer": "layer.24.mlp.gate_proj", + "ppl": 3.6199228763580322 + }, + { + "layer": "layer.24.mlp.up_proj", + "ppl": 3.620023012161255 + }, + { + "layer": "layer.24.mlp.down_proj", + "ppl": 3.619859218597412 + }, + { + "layer": "layer.25.self_attn.q_proj", + "ppl": 3.619912624359131 + }, + { + "layer": "layer.25.self_attn.k_proj", + "ppl": 3.6200687885284424 + }, + { + "layer": "layer.25.self_attn.v_proj", + "ppl": 3.6202032566070557 + }, + { + "layer": "layer.25.self_attn.o_proj", + "ppl": 3.6199653148651123 + }, + { + "layer": "layer.25.mlp.gate_proj", + "ppl": 3.6200006008148193 + }, + { + "layer": "layer.25.mlp.up_proj", + "ppl": 3.6200308799743652 + }, + { + "layer": "layer.25.mlp.down_proj", + "ppl": 3.6202402114868164 + }, + { + "layer": "layer.26.self_attn.q_proj", + "ppl": 3.6200437545776367 + }, + { + "layer": "layer.26.self_attn.k_proj", + "ppl": 3.6199653148651123 + }, + { + "layer": "layer.26.self_attn.v_proj", + "ppl": 3.6203126907348633 + }, + { + "layer": "layer.26.self_attn.o_proj", + "ppl": 3.6197872161865234 + }, + { + "layer": "layer.26.mlp.gate_proj", + "ppl": 3.619755744934082 + }, + { + "layer": "layer.26.mlp.up_proj", + "ppl": 3.6202220916748047 + }, + { + "layer": "layer.26.mlp.down_proj", + "ppl": 3.620044469833374 + }, + { + "layer": "layer.27.self_attn.q_proj", + "ppl": 3.6199307441711426 + }, + { + "layer": "layer.27.self_attn.k_proj", + "ppl": 3.6199214458465576 + }, + { + "layer": "layer.27.self_attn.v_proj", + "ppl": 3.620328187942505 + }, + { + "layer": "layer.27.self_attn.o_proj", + "ppl": 3.6199769973754883 + }, + { + "layer": "layer.27.mlp.gate_proj", + "ppl": 3.6203019618988037 + }, + { + "layer": "layer.27.mlp.up_proj", + "ppl": 3.619866132736206 + }, + { + "layer": "layer.27.mlp.down_proj", + "ppl": 3.620044469833374 + }, + { + "layer": "layer.28.self_attn.q_proj", + "ppl": 3.6200273036956787 + }, + { + "layer": "layer.28.self_attn.k_proj", + "ppl": 3.6200954914093018 + }, + { + "layer": "layer.28.self_attn.v_proj", + "ppl": 3.61926007270813 + }, + { + "layer": "layer.28.self_attn.o_proj", + "ppl": 3.6198678016662598 + }, + { + "layer": "layer.28.mlp.gate_proj", + "ppl": 3.6204326152801514 + }, + { + "layer": "layer.28.mlp.up_proj", + "ppl": 3.6201703548431396 + }, + { + "layer": "layer.28.mlp.down_proj", + "ppl": 3.6200144290924072 + }, + { + "layer": "layer.29.self_attn.q_proj", + "ppl": 3.6200592517852783 + }, + { + "layer": "layer.29.self_attn.k_proj", + "ppl": 3.6199557781219482 + }, + { + "layer": "layer.29.self_attn.v_proj", + "ppl": 3.6200435161590576 + }, + { + "layer": "layer.29.self_attn.o_proj", + "ppl": 3.6195638179779053 + }, + { + "layer": "layer.29.mlp.gate_proj", + "ppl": 3.6198534965515137 + }, + { + "layer": "layer.29.mlp.up_proj", + "ppl": 3.6202285289764404 + }, + { + "layer": "layer.29.mlp.down_proj", + "ppl": 3.620039701461792 + }, + { + "layer": "layer.30.self_attn.q_proj", + "ppl": 3.6200952529907227 + }, + { + "layer": "layer.30.self_attn.k_proj", + "ppl": 3.6198770999908447 + }, + { + "layer": "layer.30.self_attn.v_proj", + "ppl": 3.6195054054260254 + }, + { + "layer": "layer.30.self_attn.o_proj", + "ppl": 3.619797945022583 + }, + { + "layer": "layer.30.mlp.gate_proj", + "ppl": 3.6196279525756836 + }, + { + "layer": "layer.30.mlp.up_proj", + "ppl": 3.6200568675994873 + }, + { + "layer": "layer.30.mlp.down_proj", + "ppl": 3.6196789741516113 + }, + { + "layer": "layer.31.self_attn.q_proj", + "ppl": 3.6200950145721436 + }, + { + "layer": "layer.31.self_attn.k_proj", + "ppl": 3.6200506687164307 + }, + { + "layer": "layer.31.self_attn.v_proj", + "ppl": 3.6198818683624268 + }, + { + "layer": "layer.31.self_attn.o_proj", + "ppl": 3.6202759742736816 + }, + { + "layer": "layer.31.mlp.gate_proj", + "ppl": 3.6199421882629395 + }, + { + "layer": "layer.31.mlp.up_proj", + "ppl": 3.62005615234375 + }, + { + "layer": "layer.31.mlp.down_proj", + "ppl": 3.620049238204956 + }, + { + "layer": "layer.32.self_attn.q_proj", + "ppl": 3.6200883388519287 + }, + { + "layer": "layer.32.self_attn.k_proj", + "ppl": 3.620096206665039 + }, + { + "layer": "layer.32.self_attn.v_proj", + "ppl": 3.620166301727295 + }, + { + "layer": "layer.32.self_attn.o_proj", + "ppl": 3.6203105449676514 + }, + { + "layer": "layer.32.mlp.gate_proj", + "ppl": 3.620194911956787 + }, + { + "layer": "layer.32.mlp.up_proj", + "ppl": 3.6199188232421875 + }, + { + "layer": "layer.32.mlp.down_proj", + "ppl": 3.6199452877044678 + }, + { + "layer": "layer.33.self_attn.q_proj", + "ppl": 3.620354652404785 + }, + { + "layer": "layer.33.self_attn.k_proj", + "ppl": 3.619983434677124 + }, + { + "layer": "layer.33.self_attn.v_proj", + "ppl": 3.620265245437622 + }, + { + "layer": "layer.33.self_attn.o_proj", + "ppl": 3.6200389862060547 + }, + { + "layer": "layer.33.mlp.gate_proj", + "ppl": 3.6200311183929443 + }, + { + "layer": "layer.33.mlp.up_proj", + "ppl": 3.619921922683716 + }, + { + "layer": "layer.33.mlp.down_proj", + "ppl": 3.6200437545776367 + }, + { + "layer": "layer.34.self_attn.q_proj", + "ppl": 3.6201438903808594 + }, + { + "layer": "layer.34.self_attn.k_proj", + "ppl": 3.6202845573425293 + }, + { + "layer": "layer.34.self_attn.v_proj", + "ppl": 3.6204633712768555 + }, + { + "layer": "layer.34.self_attn.o_proj", + "ppl": 3.619843006134033 + }, + { + "layer": "layer.34.mlp.gate_proj", + "ppl": 3.6201188564300537 + }, + { + "layer": "layer.34.mlp.up_proj", + "ppl": 3.6198177337646484 + }, + { + "layer": "layer.34.mlp.down_proj", + "ppl": 3.6197421550750732 + }, + { + "layer": "layer.35.self_attn.q_proj", + "ppl": 3.619915246963501 + }, + { + "layer": "layer.35.self_attn.k_proj", + "ppl": 3.6200456619262695 + }, + { + "layer": "layer.35.self_attn.v_proj", + "ppl": 3.619746446609497 + }, + { + "layer": "layer.35.self_attn.o_proj", + "ppl": 3.6197824478149414 + }, + { + "layer": "layer.35.mlp.gate_proj", + "ppl": 3.619921922683716 + }, + { + "layer": "layer.35.mlp.up_proj", + "ppl": 3.6203434467315674 + }, + { + "layer": "layer.35.mlp.down_proj", + "ppl": 3.6199119091033936 + }, + { + "layer": "layer.36.self_attn.q_proj", + "ppl": 3.619969367980957 + }, + { + "layer": "layer.36.self_attn.k_proj", + "ppl": 3.620029926300049 + }, + { + "layer": "layer.36.self_attn.v_proj", + "ppl": 3.6200990676879883 + }, + { + "layer": "layer.36.self_attn.o_proj", + "ppl": 3.619624376296997 + }, + { + "layer": "layer.36.mlp.gate_proj", + "ppl": 3.6202259063720703 + }, + { + "layer": "layer.36.mlp.up_proj", + "ppl": 3.620231866836548 + }, + { + "layer": "layer.36.mlp.down_proj", + "ppl": 3.620081663131714 + }, + { + "layer": "layer.37.self_attn.q_proj", + "ppl": 3.620086431503296 + }, + { + "layer": "layer.37.self_attn.k_proj", + "ppl": 3.6200575828552246 + }, + { + "layer": "layer.37.self_attn.v_proj", + "ppl": 3.619576930999756 + }, + { + "layer": "layer.37.self_attn.o_proj", + "ppl": 3.620140314102173 + }, + { + "layer": "layer.37.mlp.gate_proj", + "ppl": 3.6200647354125977 + }, + { + "layer": "layer.37.mlp.up_proj", + "ppl": 3.6199488639831543 + }, + { + "layer": "layer.37.mlp.down_proj", + "ppl": 3.620382785797119 + }, + { + "layer": "layer.38.self_attn.q_proj", + "ppl": 3.6199376583099365 + }, + { + "layer": "layer.38.self_attn.k_proj", + "ppl": 3.6201179027557373 + }, + { + "layer": "layer.38.self_attn.v_proj", + "ppl": 3.6202402114868164 + }, + { + "layer": "layer.38.self_attn.o_proj", + "ppl": 3.6198806762695312 + }, + { + "layer": "layer.38.mlp.gate_proj", + "ppl": 3.6197640895843506 + }, + { + "layer": "layer.38.mlp.up_proj", + "ppl": 3.620088577270508 + }, + { + "layer": "layer.38.mlp.down_proj", + "ppl": 3.620137929916382 + }, + { + "layer": "layer.39.self_attn.q_proj", + "ppl": 3.620145797729492 + }, + { + "layer": "layer.39.self_attn.k_proj", + "ppl": 3.620063304901123 + }, + { + "layer": "layer.39.self_attn.v_proj", + "ppl": 3.6200273036956787 + }, + { + "layer": "layer.39.self_attn.o_proj", + "ppl": 3.6200973987579346 + }, + { + "layer": "layer.39.mlp.gate_proj", + "ppl": 3.6195571422576904 + }, + { + "layer": "layer.39.mlp.up_proj", + "ppl": 3.619696617126465 + }, + { + "layer": "layer.39.mlp.down_proj", + "ppl": 3.6199560165405273 + }, + { + "layer": "layer.40.self_attn.q_proj", + "ppl": 3.6200788021087646 + }, + { + "layer": "layer.40.self_attn.k_proj", + "ppl": 3.6201274394989014 + }, + { + "layer": "layer.40.self_attn.v_proj", + "ppl": 3.6197168827056885 + }, + { + "layer": "layer.40.self_attn.o_proj", + "ppl": 3.6199662685394287 + }, + { + "layer": "layer.40.mlp.gate_proj", + "ppl": 3.6204841136932373 + }, + { + "layer": "layer.40.mlp.up_proj", + "ppl": 3.6202173233032227 + }, + { + "layer": "layer.40.mlp.down_proj", + "ppl": 3.6197988986968994 + }, + { + "layer": "layer.41.self_attn.q_proj", + "ppl": 3.620152711868286 + }, + { + "layer": "layer.41.self_attn.k_proj", + "ppl": 3.6199870109558105 + }, + { + "layer": "layer.41.self_attn.v_proj", + "ppl": 3.620086193084717 + }, + { + "layer": "layer.41.self_attn.o_proj", + "ppl": 3.6199722290039062 + }, + { + "layer": "layer.41.mlp.gate_proj", + "ppl": 3.6201744079589844 + }, + { + "layer": "layer.41.mlp.up_proj", + "ppl": 3.6197495460510254 + }, + { + "layer": "layer.41.mlp.down_proj", + "ppl": 3.619894504547119 + }, + { + "layer": "layer.42.self_attn.q_proj", + "ppl": 3.6200408935546875 + }, + { + "layer": "layer.42.self_attn.k_proj", + "ppl": 3.6203136444091797 + }, + { + "layer": "layer.42.self_attn.v_proj", + "ppl": 3.61999773979187 + }, + { + "layer": "layer.42.self_attn.o_proj", + "ppl": 3.6199100017547607 + }, + { + "layer": "layer.42.mlp.gate_proj", + "ppl": 3.6200170516967773 + }, + { + "layer": "layer.42.mlp.up_proj", + "ppl": 3.6199324131011963 + }, + { + "layer": "layer.42.mlp.down_proj", + "ppl": 3.619938373565674 + }, + { + "layer": "layer.43.self_attn.q_proj", + "ppl": 3.6201412677764893 + }, + { + "layer": "layer.43.self_attn.k_proj", + "ppl": 3.6202552318573 + }, + { + "layer": "layer.43.self_attn.v_proj", + "ppl": 3.620271682739258 + }, + { + "layer": "layer.43.self_attn.o_proj", + "ppl": 3.619760751724243 + }, + { + "layer": "layer.43.mlp.gate_proj", + "ppl": 3.6198976039886475 + }, + { + "layer": "layer.43.mlp.up_proj", + "ppl": 3.6198995113372803 + }, + { + "layer": "layer.43.mlp.down_proj", + "ppl": 3.619694471359253 + }, + { + "layer": "layer.44.self_attn.q_proj", + "ppl": 3.6201703548431396 + }, + { + "layer": "layer.44.self_attn.k_proj", + "ppl": 3.620208501815796 + }, + { + "layer": "layer.44.self_attn.v_proj", + "ppl": 3.619811773300171 + }, + { + "layer": "layer.44.self_attn.o_proj", + "ppl": 3.62011981010437 + }, + { + "layer": "layer.44.mlp.gate_proj", + "ppl": 3.619702100753784 + }, + { + "layer": "layer.44.mlp.up_proj", + "ppl": 3.620121479034424 + }, + { + "layer": "layer.44.mlp.down_proj", + "ppl": 3.619741678237915 + }, + { + "layer": "layer.45.self_attn.q_proj", + "ppl": 3.620192527770996 + }, + { + "layer": "layer.45.self_attn.k_proj", + "ppl": 3.62011456489563 + }, + { + "layer": "layer.45.self_attn.v_proj", + "ppl": 3.6198949813842773 + }, + { + "layer": "layer.45.self_attn.o_proj", + "ppl": 3.6202242374420166 + }, + { + "layer": "layer.45.mlp.gate_proj", + "ppl": 3.619788408279419 + }, + { + "layer": "layer.45.mlp.up_proj", + "ppl": 3.6200876235961914 + }, + { + "layer": "layer.45.mlp.down_proj", + "ppl": 3.619734764099121 + }, + { + "layer": "layer.46.self_attn.q_proj", + "ppl": 3.6200978755950928 + }, + { + "layer": "layer.46.self_attn.k_proj", + "ppl": 3.620069742202759 + }, + { + "layer": "layer.46.self_attn.v_proj", + "ppl": 3.6200015544891357 + }, + { + "layer": "layer.46.self_attn.o_proj", + "ppl": 3.620175361633301 + }, + { + "layer": "layer.46.mlp.gate_proj", + "ppl": 3.6198878288269043 + }, + { + "layer": "layer.46.mlp.up_proj", + "ppl": 3.6200437545776367 + }, + { + "layer": "layer.46.mlp.down_proj", + "ppl": 3.6199488639831543 + }, + { + "layer": "layer.47.self_attn.q_proj", + "ppl": 3.6202640533447266 + }, + { + "layer": "layer.47.self_attn.k_proj", + "ppl": 3.620112895965576 + }, + { + "layer": "layer.47.self_attn.v_proj", + "ppl": 3.620100259780884 + }, + { + "layer": "layer.47.self_attn.o_proj", + "ppl": 3.6201205253601074 + }, + { + "layer": "layer.47.mlp.gate_proj", + "ppl": 3.6201467514038086 + }, + { + "layer": "layer.47.mlp.up_proj", + "ppl": 3.619774341583252 + }, + { + "layer": "layer.47.mlp.down_proj", + "ppl": 3.6200716495513916 + }, + { + "layer": "layer.48.self_attn.q_proj", + "ppl": 3.620192527770996 + }, + { + "layer": "layer.48.self_attn.k_proj", + "ppl": 3.62011456489563 + }, + { + "layer": "layer.48.self_attn.v_proj", + "ppl": 3.6200389862060547 + }, + { + "layer": "layer.48.self_attn.o_proj", + "ppl": 3.6200459003448486 + }, + { + "layer": "layer.48.mlp.gate_proj", + "ppl": 3.6198623180389404 + }, + { + "layer": "layer.48.mlp.up_proj", + "ppl": 3.620157480239868 + }, + { + "layer": "layer.48.mlp.down_proj", + "ppl": 3.6196820735931396 + }, + { + "layer": "layer.49.self_attn.q_proj", + "ppl": 3.6201322078704834 + }, + { + "layer": "layer.49.self_attn.k_proj", + "ppl": 3.6203322410583496 + }, + { + "layer": "layer.49.self_attn.v_proj", + "ppl": 3.620173931121826 + }, + { + "layer": "layer.49.self_attn.o_proj", + "ppl": 3.620140314102173 + }, + { + "layer": "layer.49.mlp.gate_proj", + "ppl": 3.6201655864715576 + }, + { + "layer": "layer.49.mlp.up_proj", + "ppl": 3.6204020977020264 + }, + { + "layer": "layer.49.mlp.down_proj", + "ppl": 3.620009422302246 + }, + { + "layer": "layer.50.self_attn.q_proj", + "ppl": 3.620196580886841 + }, + { + "layer": "layer.50.self_attn.k_proj", + "ppl": 3.6200647354125977 + }, + { + "layer": "layer.50.self_attn.v_proj", + "ppl": 3.620164632797241 + }, + { + "layer": "layer.50.self_attn.o_proj", + "ppl": 3.6202871799468994 + }, + { + "layer": "layer.50.mlp.gate_proj", + "ppl": 3.620363473892212 + }, + { + "layer": "layer.50.mlp.up_proj", + "ppl": 3.619579553604126 + }, + { + "layer": "layer.50.mlp.down_proj", + "ppl": 3.619896173477173 + }, + { + "layer": "layer.51.self_attn.q_proj", + "ppl": 3.620088577270508 + }, + { + "layer": "layer.51.self_attn.k_proj", + "ppl": 3.620227098464966 + }, + { + "layer": "layer.51.self_attn.v_proj", + "ppl": 3.6199400424957275 + }, + { + "layer": "layer.51.self_attn.o_proj", + "ppl": 3.620255947113037 + }, + { + "layer": "layer.51.mlp.gate_proj", + "ppl": 3.6198465824127197 + }, + { + "layer": "layer.51.mlp.up_proj", + "ppl": 3.6197030544281006 + }, + { + "layer": "layer.51.mlp.down_proj", + "ppl": 3.6201212406158447 + }, + { + "layer": "layer.52.self_attn.q_proj", + "ppl": 3.6200671195983887 + }, + { + "layer": "layer.52.self_attn.k_proj", + "ppl": 3.6202218532562256 + }, + { + "layer": "layer.52.self_attn.v_proj", + "ppl": 3.6201281547546387 + }, + { + "layer": "layer.52.self_attn.o_proj", + "ppl": 3.6201343536376953 + }, + { + "layer": "layer.52.mlp.gate_proj", + "ppl": 3.620227098464966 + }, + { + "layer": "layer.52.mlp.up_proj", + "ppl": 3.61997652053833 + }, + { + "layer": "layer.52.mlp.down_proj", + "ppl": 3.6198160648345947 + }, + { + "layer": "layer.53.self_attn.q_proj", + "ppl": 3.620187759399414 + }, + { + "layer": "layer.53.self_attn.k_proj", + "ppl": 3.620208501815796 + }, + { + "layer": "layer.53.self_attn.v_proj", + "ppl": 3.620131015777588 + }, + { + "layer": "layer.53.self_attn.o_proj", + "ppl": 3.6201705932617188 + }, + { + "layer": "layer.53.mlp.gate_proj", + "ppl": 3.61968994140625 + }, + { + "layer": "layer.53.mlp.up_proj", + "ppl": 3.620213031768799 + }, + { + "layer": "layer.53.mlp.down_proj", + "ppl": 3.619856119155884 + }, + { + "layer": "layer.54.self_attn.q_proj", + "ppl": 3.620147943496704 + }, + { + "layer": "layer.54.self_attn.k_proj", + "ppl": 3.620181083679199 + }, + { + "layer": "layer.54.self_attn.v_proj", + "ppl": 3.620201587677002 + }, + { + "layer": "layer.54.self_attn.o_proj", + "ppl": 3.620060920715332 + }, + { + "layer": "layer.54.mlp.gate_proj", + "ppl": 3.6196956634521484 + }, + { + "layer": "layer.54.mlp.up_proj", + "ppl": 3.6200437545776367 + }, + { + "layer": "layer.54.mlp.down_proj", + "ppl": 3.6198408603668213 + }, + { + "layer": "layer.55.self_attn.q_proj", + "ppl": 3.62015962600708 + }, + { + "layer": "layer.55.self_attn.k_proj", + "ppl": 3.620253562927246 + }, + { + "layer": "layer.55.self_attn.v_proj", + "ppl": 3.6202855110168457 + }, + { + "layer": "layer.55.self_attn.o_proj", + "ppl": 3.620100975036621 + }, + { + "layer": "layer.55.mlp.gate_proj", + "ppl": 3.619879961013794 + }, + { + "layer": "layer.55.mlp.up_proj", + "ppl": 3.6200006008148193 + }, + { + "layer": "layer.55.mlp.down_proj", + "ppl": 3.619856595993042 + }, + { + "layer": "layer.56.self_attn.q_proj", + "ppl": 3.6200990676879883 + }, + { + "layer": "layer.56.self_attn.k_proj", + "ppl": 3.6201987266540527 + }, + { + "layer": "layer.56.self_attn.v_proj", + "ppl": 3.6201515197753906 + }, + { + "layer": "layer.56.self_attn.o_proj", + "ppl": 3.6200692653656006 + }, + { + "layer": "layer.56.mlp.gate_proj", + "ppl": 3.619800567626953 + }, + { + "layer": "layer.56.mlp.up_proj", + "ppl": 3.619539737701416 + }, + { + "layer": "layer.56.mlp.down_proj", + "ppl": 3.6200194358825684 + }, + { + "layer": "layer.57.self_attn.q_proj", + "ppl": 3.6201837062835693 + }, + { + "layer": "layer.57.self_attn.k_proj", + "ppl": 3.6200740337371826 + }, + { + "layer": "layer.57.self_attn.v_proj", + "ppl": 3.6202304363250732 + }, + { + "layer": "layer.57.self_attn.o_proj", + "ppl": 3.620100975036621 + }, + { + "layer": "layer.57.mlp.gate_proj", + "ppl": 3.620155096054077 + }, + { + "layer": "layer.57.mlp.up_proj", + "ppl": 3.6204044818878174 + }, + { + "layer": "layer.57.mlp.down_proj", + "ppl": 3.6201672554016113 + }, + { + "layer": "layer.58.self_attn.q_proj", + "ppl": 3.6203079223632812 + }, + { + "layer": "layer.58.self_attn.k_proj", + "ppl": 3.6202597618103027 + }, + { + "layer": "layer.58.self_attn.v_proj", + "ppl": 3.6202003955841064 + }, + { + "layer": "layer.58.self_attn.o_proj", + "ppl": 3.620231866836548 + }, + { + "layer": "layer.58.mlp.gate_proj", + "ppl": 3.6197266578674316 + }, + { + "layer": "layer.58.mlp.up_proj", + "ppl": 3.619652032852173 + }, + { + "layer": "layer.58.mlp.down_proj", + "ppl": 3.6198654174804688 + }, + { + "layer": "layer.59.self_attn.q_proj", + "ppl": 3.6201703548431396 + }, + { + "layer": "layer.59.self_attn.k_proj", + "ppl": 3.6202611923217773 + }, + { + "layer": "layer.59.self_attn.v_proj", + "ppl": 3.6201586723327637 + }, + { + "layer": "layer.59.self_attn.o_proj", + "ppl": 3.6200287342071533 + }, + { + "layer": "layer.59.mlp.gate_proj", + "ppl": 3.6202454566955566 + }, + { + "layer": "layer.59.mlp.up_proj", + "ppl": 3.6199870109558105 + }, + { + "layer": "layer.59.mlp.down_proj", + "ppl": 3.6197562217712402 + }, + { + "layer": "layer.60.self_attn.q_proj", + "ppl": 3.620163679122925 + }, + { + "layer": "layer.60.self_attn.k_proj", + "ppl": 3.6200203895568848 + }, + { + "layer": "layer.60.self_attn.v_proj", + "ppl": 3.620327949523926 + }, + { + "layer": "layer.60.self_attn.o_proj", + "ppl": 3.6201891899108887 + }, + { + "layer": "layer.60.mlp.gate_proj", + "ppl": 3.6200692653656006 + }, + { + "layer": "layer.60.mlp.up_proj", + "ppl": 3.6198012828826904 + }, + { + "layer": "layer.60.mlp.down_proj", + "ppl": 3.6200931072235107 + }, + { + "layer": "layer.61.self_attn.q_proj", + "ppl": 3.6201939582824707 + }, + { + "layer": "layer.61.self_attn.k_proj", + "ppl": 3.620049238204956 + }, + { + "layer": "layer.61.self_attn.v_proj", + "ppl": 3.6201603412628174 + }, + { + "layer": "layer.61.self_attn.o_proj", + "ppl": 3.619960069656372 + }, + { + "layer": "layer.61.mlp.gate_proj", + "ppl": 3.620267629623413 + }, + { + "layer": "layer.61.mlp.up_proj", + "ppl": 3.6200153827667236 + }, + { + "layer": "layer.61.mlp.down_proj", + "ppl": 3.62024188041687 + }, + { + "layer": "layer.62.self_attn.q_proj", + "ppl": 3.62011981010437 + }, + { + "layer": "layer.62.self_attn.k_proj", + "ppl": 3.6200661659240723 + }, + { + "layer": "layer.62.self_attn.v_proj", + "ppl": 3.6203157901763916 + }, + { + "layer": "layer.62.self_attn.o_proj", + "ppl": 3.6201460361480713 + }, + { + "layer": "layer.62.mlp.gate_proj", + "ppl": 3.6200671195983887 + }, + { + "layer": "layer.62.mlp.up_proj", + "ppl": 3.6196866035461426 + }, + { + "layer": "layer.62.mlp.down_proj", + "ppl": 3.619919538497925 + }, + { + "layer": "layer.63.self_attn.q_proj", + "ppl": 3.620083808898926 + }, + { + "layer": "layer.63.self_attn.k_proj", + "ppl": 3.61999773979187 + }, + { + "layer": "layer.63.self_attn.v_proj", + "ppl": 3.6199049949645996 + }, + { + "layer": "layer.63.self_attn.o_proj", + "ppl": 3.620037794113159 + }, + { + "layer": "layer.63.mlp.gate_proj", + "ppl": 3.619645118713379 + }, + { + "layer": "layer.63.mlp.up_proj", + "ppl": 3.620074987411499 + }, + { + "layer": "layer.63.mlp.down_proj", + "ppl": 3.6200153827667236 + }, + { + "layer": "layer.64.self_attn.q_proj", + "ppl": 3.620062828063965 + }, + { + "layer": "layer.64.self_attn.k_proj", + "ppl": 3.6200110912323 + }, + { + "layer": "layer.64.self_attn.v_proj", + "ppl": 3.6199851036071777 + }, + { + "layer": "layer.64.self_attn.o_proj", + "ppl": 3.620034694671631 + }, + { + "layer": "layer.64.mlp.gate_proj", + "ppl": 3.619999885559082 + }, + { + "layer": "layer.64.mlp.up_proj", + "ppl": 3.6200835704803467 + }, + { + "layer": "layer.64.mlp.down_proj", + "ppl": 3.62007212638855 + }, + { + "layer": "layer.65.self_attn.q_proj", + "ppl": 3.6200954914093018 + }, + { + "layer": "layer.65.self_attn.k_proj", + "ppl": 3.6201202869415283 + }, + { + "layer": "layer.65.self_attn.v_proj", + "ppl": 3.619912624359131 + }, + { + "layer": "layer.65.self_attn.o_proj", + "ppl": 3.6199722290039062 + }, + { + "layer": "layer.65.mlp.gate_proj", + "ppl": 3.6196727752685547 + }, + { + "layer": "layer.65.mlp.up_proj", + "ppl": 3.619673490524292 + }, + { + "layer": "layer.65.mlp.down_proj", + "ppl": 3.619544267654419 + }, + { + "layer": "layer.66.self_attn.q_proj", + "ppl": 3.620147466659546 + }, + { + "layer": "layer.66.self_attn.k_proj", + "ppl": 3.620234251022339 + }, + { + "layer": "layer.66.self_attn.v_proj", + "ppl": 3.620182752609253 + }, + { + "layer": "layer.66.self_attn.o_proj", + "ppl": 3.6201846599578857 + }, + { + "layer": "layer.66.mlp.gate_proj", + "ppl": 3.6199214458465576 + }, + { + "layer": "layer.66.mlp.up_proj", + "ppl": 3.619497537612915 + }, + { + "layer": "layer.66.mlp.down_proj", + "ppl": 3.6199071407318115 + }, + { + "layer": "layer.67.self_attn.q_proj", + "ppl": 3.620100975036621 + }, + { + "layer": "layer.67.self_attn.k_proj", + "ppl": 3.6200308799743652 + }, + { + "layer": "layer.67.self_attn.v_proj", + "ppl": 3.6200950145721436 + }, + { + "layer": "layer.67.self_attn.o_proj", + "ppl": 3.6202285289764404 + }, + { + "layer": "layer.67.mlp.gate_proj", + "ppl": 3.6202337741851807 + }, + { + "layer": "layer.67.mlp.up_proj", + "ppl": 3.6201529502868652 + }, + { + "layer": "layer.67.mlp.down_proj", + "ppl": 3.6202950477600098 + }, + { + "layer": "layer.68.self_attn.q_proj", + "ppl": 3.62011456489563 + }, + { + "layer": "layer.68.self_attn.k_proj", + "ppl": 3.620065212249756 + }, + { + "layer": "layer.68.self_attn.v_proj", + "ppl": 3.620129346847534 + }, + { + "layer": "layer.68.self_attn.o_proj", + "ppl": 3.6199586391448975 + }, + { + "layer": "layer.68.mlp.gate_proj", + "ppl": 3.619983434677124 + }, + { + "layer": "layer.68.mlp.up_proj", + "ppl": 3.619842767715454 + }, + { + "layer": "layer.68.mlp.down_proj", + "ppl": 3.6199870109558105 + }, + { + "layer": "layer.69.self_attn.q_proj", + "ppl": 3.6202070713043213 + }, + { + "layer": "layer.69.self_attn.k_proj", + "ppl": 3.6201086044311523 + }, + { + "layer": "layer.69.self_attn.v_proj", + "ppl": 3.619751214981079 + }, + { + "layer": "layer.69.self_attn.o_proj", + "ppl": 3.62001633644104 + }, + { + "layer": "layer.69.mlp.gate_proj", + "ppl": 3.61968994140625 + }, + { + "layer": "layer.69.mlp.up_proj", + "ppl": 3.61979603767395 + }, + { + "layer": "layer.69.mlp.down_proj", + "ppl": 3.6198203563690186 + }, + { + "layer": "layer.70.self_attn.q_proj", + "ppl": 3.620147466659546 + }, + { + "layer": "layer.70.self_attn.k_proj", + "ppl": 3.6202540397644043 + }, + { + "layer": "layer.70.self_attn.v_proj", + "ppl": 3.6199207305908203 + }, + { + "layer": "layer.70.self_attn.o_proj", + "ppl": 3.6201508045196533 + }, + { + "layer": "layer.70.mlp.gate_proj", + "ppl": 3.6201138496398926 + }, + { + "layer": "layer.70.mlp.up_proj", + "ppl": 3.6200335025787354 + }, + { + "layer": "layer.70.mlp.down_proj", + "ppl": 3.6200661659240723 + }, + { + "layer": "layer.71.self_attn.q_proj", + "ppl": 3.6200766563415527 + }, + { + "layer": "layer.71.self_attn.k_proj", + "ppl": 3.6201059818267822 + }, + { + "layer": "layer.71.self_attn.v_proj", + "ppl": 3.609890937805176 + }, + { + "layer": "layer.71.self_attn.o_proj", + "ppl": 3.620072841644287 + }, + { + "layer": "layer.71.mlp.gate_proj", + "ppl": 3.619668483734131 + }, + { + "layer": "layer.71.mlp.up_proj", + "ppl": 3.620453357696533 + }, + { + "layer": "layer.71.mlp.down_proj", + "ppl": 3.620140314102173 + }, + { + "layer": "layer.72.self_attn.q_proj", + "ppl": 3.6199586391448975 + }, + { + "layer": "layer.72.self_attn.k_proj", + "ppl": 3.6200485229492188 + }, + { + "layer": "layer.72.self_attn.v_proj", + "ppl": 3.6184959411621094 + }, + { + "layer": "layer.72.self_attn.o_proj", + "ppl": 3.6201059818267822 + }, + { + "layer": "layer.72.mlp.gate_proj", + "ppl": 3.619974136352539 + }, + { + "layer": "layer.72.mlp.up_proj", + "ppl": 3.619962692260742 + }, + { + "layer": "layer.72.mlp.down_proj", + "ppl": 3.620361566543579 + }, + { + "layer": "layer.73.self_attn.q_proj", + "ppl": 3.6200544834136963 + }, + { + "layer": "layer.73.self_attn.k_proj", + "ppl": 3.6203479766845703 + }, + { + "layer": "layer.73.self_attn.v_proj", + "ppl": 3.618893623352051 + }, + { + "layer": "layer.73.self_attn.o_proj", + "ppl": 3.619778871536255 + }, + { + "layer": "layer.73.mlp.gate_proj", + "ppl": 3.619912624359131 + }, + { + "layer": "layer.73.mlp.up_proj", + "ppl": 3.62068247795105 + }, + { + "layer": "layer.73.mlp.down_proj", + "ppl": 3.620199203491211 + }, + { + "layer": "layer.74.self_attn.q_proj", + "ppl": 3.6201791763305664 + }, + { + "layer": "layer.74.self_attn.k_proj", + "ppl": 3.6202001571655273 + }, + { + "layer": "layer.74.self_attn.v_proj", + "ppl": 3.618903160095215 + }, + { + "layer": "layer.74.self_attn.o_proj", + "ppl": 3.6199142932891846 + }, + { + "layer": "layer.74.mlp.gate_proj", + "ppl": 3.6199488639831543 + }, + { + "layer": "layer.74.mlp.up_proj", + "ppl": 3.619795799255371 + }, + { + "layer": "layer.74.mlp.down_proj", + "ppl": 3.6198794841766357 + }, + { + "layer": "layer.75.self_attn.q_proj", + "ppl": 3.6199662685394287 + }, + { + "layer": "layer.75.self_attn.k_proj", + "ppl": 3.622318744659424 + }, + { + "layer": "layer.75.self_attn.v_proj", + "ppl": 3.6071560382843018 + }, + { + "layer": "layer.75.self_attn.o_proj", + "ppl": 3.619859218597412 + }, + { + "layer": "layer.75.mlp.gate_proj", + "ppl": 3.619819402694702 + }, + { + "layer": "layer.75.mlp.up_proj", + "ppl": 3.6199769973754883 + }, + { + "layer": "layer.75.mlp.down_proj", + "ppl": 3.6210978031158447 + }, + { + "layer": "layer.76.self_attn.q_proj", + "ppl": 3.6200761795043945 + }, + { + "layer": "layer.76.self_attn.k_proj", + "ppl": 3.620743751525879 + }, + { + "layer": "layer.76.self_attn.v_proj", + "ppl": 3.6200082302093506 + }, + { + "layer": "layer.76.self_attn.o_proj", + "ppl": 3.6195027828216553 + }, + { + "layer": "layer.76.mlp.gate_proj", + "ppl": 3.620293140411377 + }, + { + "layer": "layer.76.mlp.up_proj", + "ppl": 3.619781494140625 + }, + { + "layer": "layer.76.mlp.down_proj", + "ppl": 3.6203882694244385 + }, + { + "layer": "layer.77.self_attn.q_proj", + "ppl": 3.620105028152466 + }, + { + "layer": "layer.77.self_attn.k_proj", + "ppl": 3.620332956314087 + }, + { + "layer": "layer.77.self_attn.v_proj", + "ppl": 3.618912935256958 + }, + { + "layer": "layer.77.self_attn.o_proj", + "ppl": 3.6200153827667236 + }, + { + "layer": "layer.77.mlp.gate_proj", + "ppl": 3.619969367980957 + }, + { + "layer": "layer.77.mlp.up_proj", + "ppl": 3.6199166774749756 + }, + { + "layer": "layer.77.mlp.down_proj", + "ppl": 3.61982798576355 + }, + { + "layer": "layer.78.self_attn.q_proj", + "ppl": 3.6200640201568604 + }, + { + "layer": "layer.78.self_attn.k_proj", + "ppl": 3.620117664337158 + }, + { + "layer": "layer.78.self_attn.v_proj", + "ppl": 3.6202454566955566 + }, + { + "layer": "layer.78.self_attn.o_proj", + "ppl": 3.6199135780334473 + }, + { + "layer": "layer.78.mlp.gate_proj", + "ppl": 3.6201164722442627 + }, + { + "layer": "layer.78.mlp.up_proj", + "ppl": 3.6202518939971924 + }, + { + "layer": "layer.78.mlp.down_proj", + "ppl": 3.6204943656921387 + }, + { + "layer": "layer.79.self_attn.q_proj", + "ppl": 3.620296001434326 + }, + { + "layer": "layer.79.self_attn.k_proj", + "ppl": 3.620213031768799 + }, + { + "layer": "layer.79.self_attn.v_proj", + "ppl": 3.6201250553131104 + }, + { + "layer": "layer.79.self_attn.o_proj", + "ppl": 3.6202456951141357 + }, + { + "layer": "layer.79.mlp.gate_proj", + "ppl": 3.6197052001953125 + }, + { + "layer": "layer.79.mlp.up_proj", + "ppl": 3.6199381351470947 + }, + { + "layer": "layer.79.mlp.down_proj", + "ppl": 3.6188547611236572 + } + ] + } + ] +} \ No newline at end of file