added model trained on 4M events per dataset

clic/clusters/v2.3.0/README.md

@@ -3,4 +3,5 @@ pyg-clic_20250204_081614_352844 - legacy GNNLSH model, full dataset, 2 epochs /
 pyg-clic_20250211_145811_219129 - transformer, full dataset, 2 epochs / ~45 hours
 pyg-clic_20250130_214007_333962 - transformer + flash attention, full dataset, 10 epochs / ~80 hours
 pyg-clic_20250208_095515_010468 - transformer + flash attention, 1M events from each dataset, 10 epochs / ~12 hours
+pyg-clic_20250209_100514_187330 - transformer + flash attention, 4M events from each dataset, 10 epochs / ~40 hours
 ```

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/.gitattributes

@@ -0,0 +1,4 @@
+plots_checkpoint*/** filter=lfs diff=lfs merge=lfs -text
+preds_checkpoint*/** filter=lfs diff=lfs merge=lfs -text
+runs/** filter=lfs diff=lfs merge=lfs -text
+checkpoints/** filter=lfs diff=lfs merge=lfs -text

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/checkpoints/checkpoint-10-2.022640.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7b4a648f3b8a5c469daf46d1f04a4e1981132f0c830a18e0330ae5e9ea9bf0f
+size 631766830

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_1.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.14345213541666665, "Regression_eta": 0.00018138920084635418, "Regression_sin_phi": 0.00012380649820963543, "Regression_cos_phi": 0.0001234324747721354, "Regression_energy": 0.14386429166666667, "Classification_binary": 2.556795166666667, "Classification": 0.021014436197916667, "ispu": 1.9530105590820312e-05, "MET": 5.339154, "Sliced_Wasserstein_Loss": 46.48268266666667, "Total": 2.8655763333333333}, "valid": {"Regression_pt": 0.12151891304947443, "Regression_eta": 0.00016556585027634096, "Regression_sin_phi": 0.00010927752452673626, "Regression_cos_phi": 0.00010814136957323869, "Regression_energy": 0.12153660913093774, "Classification_binary": 2.2896245954584358, "Classification": 0.01716714167041397, "ispu": 3.655637660323791e-14, "MET": 4.209416707900931, "Sliced_Wasserstein_Loss": 42.2115217449449, "Total": 2.5502346173906068}, "epoch_train_time": 14776.063064575195, "epoch_valid_time": 1052.652112007141, "epoch_total_time": 15828.715177297592}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_10.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.08632264583333334, "Regression_eta": 0.00012097669474283854, "Regression_sin_phi": 7.552708943684896e-05, "Regression_cos_phi": 7.548623657226563e-05, "Regression_energy": 0.08628893229166666, "Classification_binary": 1.7798005, "Classification": 0.011805893229166666, "ispu": 1.4743171128372978e-15, "MET": 3.1923203333333334, "Sliced_Wasserstein_Loss": 35.96093333333334, "Total": 1.9644896666666667}, "valid": {"Regression_pt": 0.0918824169436049, "Regression_eta": 0.00012829951121741545, "Regression_sin_phi": 8.007459644584129e-05, "Regression_cos_phi": 8.001636282905635e-05, "Regression_energy": 0.09222343545221728, "Classification_binary": 1.8263311005536536, "Classification": 0.011915614825079655, "ispu": 3.528617966985692e-15, "MET": 3.189476110316144, "Sliced_Wasserstein_Loss": 35.713200090938265, "Total": 2.022640118487215}, "epoch_train_time": 14746.338783025742, "epoch_valid_time": 1048.431301832199, "epoch_total_time": 15794.770089626312}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_2.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.11259471875, "Regression_eta": 0.0001522861124674479, "Regression_sin_phi": 9.933065795898437e-05, "Regression_cos_phi": 9.910172526041666e-05, "Regression_energy": 0.11210575, "Classification_binary": 2.1718713333333333, "Classification": 0.016470404947916666, "ispu": 1.6110830832379482e-13, "MET": 3.9014626666666667, "Sliced_Wasserstein_Loss": 41.21674933333333, "Total": 2.4133845}, "valid": {"Regression_pt": 0.11062613254921365, "Regression_eta": 0.00015335334511737818, "Regression_sin_phi": 9.96017399938193e-05, "Regression_cos_phi": 9.946891607747967e-05, "Regression_energy": 0.11057723651305232, "Classification_binary": 2.1408815997111375, "Classification": 0.015836920370402102, "ispu": 9.730737191254473e-14, "MET": 3.7601486439499303, "Sliced_Wasserstein_Loss": 40.27225981598374, "Total": 2.3782706015299024}, "epoch_train_time": 14780.387635231018, "epoch_valid_time": 1050.3413667678833, "epoch_total_time": 15830.729005336761}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_3.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.10544566666666666, "Regression_eta": 0.00014501154581705728, "Regression_sin_phi": 9.309078979492188e-05, "Regression_cos_phi": 9.300138346354167e-05, "Regression_energy": 0.1050540625, "Classification_binary": 2.0617886666666667, "Classification": 0.015343759114583333, "ispu": 2.68790264120374e-13, "MET": 3.6933973333333334, "Sliced_Wasserstein_Loss": 39.756704, "Total": 2.2879665}, "valid": {"Regression_pt": 0.10576827087108498, "Regression_eta": 0.00014764572370877505, "Regression_sin_phi": 9.513243106618799e-05, "Regression_cos_phi": 9.496072734678902e-05, "Regression_energy": 0.1057881544755336, "Classification_binary": 2.0440354776265113, "Classification": 0.014765050432395588, "ispu": 1.2218228283550015e-13, "MET": 3.6542022172889697, "Sliced_Wasserstein_Loss": 38.955299828822085, "Total": 2.270691590550444}, "epoch_train_time": 14783.080834150314, "epoch_valid_time": 1051.713621377945, "epoch_total_time": 15834.79445886612}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_4.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.10056632291666667, "Regression_eta": 0.0001380366007486979, "Regression_sin_phi": 8.740170288085938e-05, "Regression_cos_phi": 8.738942464192709e-05, "Regression_energy": 0.10040232291666666, "Classification_binary": 1.9751141666666667, "Classification": 0.014366264322916667, "ispu": 1.1744162028056356e-13, "MET": 3.5545613333333335, "Sliced_Wasserstein_Loss": 38.72884533333333, "Total": 2.1907466666666666}, "valid": {"Regression_pt": 0.10218279662968399, "Regression_eta": 0.00014112312500140395, "Regression_sin_phi": 8.978832975815895e-05, "Regression_cos_phi": 8.988734348656673e-05, "Regression_energy": 0.10243130577073527, "Classification_binary": 1.979122228388788, "Classification": 0.0139767471399998, "ispu": 6.654092656737552e-14, "MET": 3.607495636969081, "Sliced_Wasserstein_Loss": 37.96207004921365, "Total": 2.1980322897721196}, "epoch_train_time": 14766.17946434021, "epoch_valid_time": 1046.7795479297638, "epoch_total_time": 15812.959015369415}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_5.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.09685227083333334, "Regression_eta": 0.00013351305135091146, "Regression_sin_phi": 8.390215047200521e-05, "Regression_cos_phi": 8.392649332682291e-05, "Regression_energy": 0.09670648958333333, "Classification_binary": 1.925606, "Classification": 0.013763524739583333, "ispu": 4.683713920409597e-14, "MET": 3.4562666666666666, "Sliced_Wasserstein_Loss": 38.02410666666667, "Total": 2.1332263333333334}, "valid": {"Regression_pt": 0.09853944118308347, "Regression_eta": 0.00013799951511920551, "Regression_sin_phi": 8.69795270331903e-05, "Regression_cos_phi": 8.684615470565448e-05, "Regression_energy": 0.09891279585236239, "Classification_binary": 1.9436084522440356, "Classification": 0.013558235629632586, "ispu": 5.780081176356482e-15, "MET": 3.348571692454798, "Sliced_Wasserstein_Loss": 37.41128838129881, "Total": 2.154925627875254}, "epoch_train_time": 14815.98518037796, "epoch_valid_time": 1051.0379424095154, "epoch_total_time": 15867.023137331009}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_6.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.09374578125, "Regression_eta": 0.00012966026814778645, "Regression_sin_phi": 8.1197021484375e-05, "Regression_cos_phi": 8.118624369303386e-05, "Regression_energy": 0.09360591666666666, "Classification_binary": 1.8861036666666666, "Classification": 0.013247291666666666, "ispu": 2.0679910524753118e-14, "MET": 3.374214333333333, "Sliced_Wasserstein_Loss": 37.44024266666667, "Total": 2.087004}, "valid": {"Regression_pt": 0.09613845513391663, "Regression_eta": 0.00013378030720452924, "Regression_sin_phi": 8.459722242011506e-05, "Regression_cos_phi": 8.435377886400664e-05, "Regression_energy": 0.09640644302441626, "Classification_binary": 1.9046849089279982, "Classification": 0.013058696798657407, "ispu": 5.780081176356482e-15, "MET": 3.427107293784102, "Sliced_Wasserstein_Loss": 36.837447844228095, "Total": 2.110593848627902}, "epoch_train_time": 14761.964252233505, "epoch_valid_time": 1050.0273101329803, "epoch_total_time": 15811.99156332016}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_7.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.09120145833333333, "Regression_eta": 0.00012652694702148439, "Regression_sin_phi": 7.902403767903645e-05, "Regression_cos_phi": 7.899856567382812e-05, "Regression_energy": 0.091082625, "Classification_binary": 1.8449146666666667, "Classification": 0.01261265234375, "ispu": 2.9779337514431366e-15, "MET": 3.3050646666666665, "Sliced_Wasserstein_Loss": 36.90396533333333, "Total": 2.0401078333333333}, "valid": {"Regression_pt": 0.09425217691839628, "Regression_eta": 0.0001316729542838452, "Regression_sin_phi": 8.258784379731579e-05, "Regression_cos_phi": 8.238991837049022e-05, "Regression_energy": 0.09451814706381392, "Classification_binary": 1.8677452073526266, "Classification": 0.012429491726487694, "ispu": 0.0, "MET": 3.2817853489087407, "Sliced_Wasserstein_Loss": 36.440167968332084, "Total": 2.069240757395421}, "epoch_train_time": 14761.517078876495, "epoch_valid_time": 1050.0858120918274, "epoch_total_time": 15811.602896690369}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_8.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.08908064583333333, "Regression_eta": 0.00012395712280273438, "Regression_sin_phi": 7.736606852213541e-05, "Regression_cos_phi": 7.733473714192708e-05, "Regression_energy": 0.08898997916666666, "Classification_binary": 1.8095385, "Classification": 0.012140442708333333, "ispu": 3.6476338986328e-15, "MET": 3.245734666666667, "Sliced_Wasserstein_Loss": 36.46280266666667, "Total": 2.000024}, "valid": {"Regression_pt": 0.09259154006633144, "Regression_eta": 0.00012971040742644987, "Regression_sin_phi": 8.117154637568645e-05, "Regression_cos_phi": 8.10459602117615e-05, "Regression_energy": 0.09286328919038193, "Classification_binary": 1.8419527572349417, "Classification": 0.012070082516240171, "ispu": 7.06212367094189e-15, "MET": 3.2130101486439497, "Sliced_Wasserstein_Loss": 36.224229699368784, "Total": 2.039768358163047}, "epoch_train_time": 14774.942404031754, "epoch_valid_time": 1049.9855282306671, "epoch_total_time": 15824.927933216095}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/history/epoch_9.json

@@ -0,0 +1 @@
+{"train": {"Regression_pt": 0.08741290625, "Regression_eta": 0.00012209530639648437, "Regression_sin_phi": 7.62167714436849e-05, "Regression_cos_phi": 7.61659444173177e-05, "Regression_energy": 0.08735395833333333, "Classification_binary": 1.7886371666666667, "Classification": 0.0119045390625, "ispu": 1.445688309805367e-15, "MET": 3.210315666666667, "Sliced_Wasserstein_Loss": 36.153306666666666, "Total": 1.9755803333333333}, "valid": {"Regression_pt": 0.09195151002034409, "Regression_eta": 0.00012858418764322726, "Regression_sin_phi": 8.051994459838476e-05, "Regression_cos_phi": 8.036921441880243e-05, "Regression_energy": 0.09229617875387824, "Classification_binary": 1.8273226035091472, "Classification": 0.011916637573699282, "ispu": 0.0, "MET": 3.1647733731411147, "Sliced_Wasserstein_Loss": 35.92370881031347, "Total": 2.0237791453808707}, "epoch_train_time": 14748.775985002518, "epoch_valid_time": 1047.9388043880463, "epoch_total_time": 15796.71479010582}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/hyperparameters.json

@@ -0,0 +1 @@
+{"num_mlpf_params": 52630547, "train": true, "test": null, "make_plots": null, "comet": true, "save_attention": true, "dataset": "clic", "sort_data": false, "data_dir": "/scratch/persistent/joosep/tensorflow_datasets", "gpus": 1, "gpu_batch_multiplier": 256, "load": null, "finetune": null, "num_epochs": 10, "patience": 20, "lr": 0.0001, "lr_schedule": "cosinedecay", "conv_type": "attention", "ntrain": 4000000, "ntest": 2000, "nvalid": null, "num_workers": 8, "prefetch_factor": 100, "checkpoint_freq": 1, "comet_name": "particleflow-pt", "comet_offline": false, "comet_step_freq": 100, "dtype": "bfloat16", "val_freq": null, "model": {"trainable": "all", "learned_representation_mode": "last", "input_encoding": "split", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "gnn_lsh": {"conv_type": "gnn_lsh", "embedding_dim": 1024, "width": 1024, "num_convs": 3, "activation": "elu", "bin_size": 32, "max_num_bins": 200, "distance_dim": 128, "layernorm": true, "num_node_messages": 2, "ffn_dist_hidden_dim": 128, "ffn_dist_num_layers": 2}, "attention": {"conv_type": "attention", "num_convs": 3, "dropout_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "activation": "relu", "head_dim": 32, "num_heads": 32, "attention_type": "flash", "use_pre_layernorm": true}, "mamba": {"conv_type": "mamba", "embedding_dim": 128, "width": 128, "num_convs": 2, "dropout": 0.0, "activation": "elu", "num_heads": 2, "d_state": 16, "d_conv": 4, "expand": 2}}, "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "raytune": {"local_dir": null, "sched": null, "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"clic": {"physical": {"batch_size": 1, "samples": {"clic_edm_qq_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ttbar_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ww_fullhad_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}}}}}, "valid_dataset": {"clic": {"physical": {"batch_size": 1, "samples": {"clic_edm_qq_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ttbar_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ww_fullhad_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}}}}}, "test_dataset": {"clic_edm_qq_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ttbar_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}, "clic_edm_ww_fullhad_pf": {"version": "2.5.0", "splits": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}}, "enabled_test_datasets": ["clic_edm_qq_pf"]}

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/model_kwargs.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2784c409dcad68c684a808b0e9c311e9eb8ee97cd7c92d4d4f1771878dd87f5
+size 553

clic/clusters/v2.3.0/pyg-clic_20250209_100514_187330/overridden_config.yaml

@@ -0,0 +1,225 @@
+checkpoint_freq: 1
+comet: true
+comet_name: particleflow-pt
+comet_offline: false
+comet_step_freq: 100
+conv_type: attention
+data_dir: /scratch/persistent/joosep/tensorflow_datasets
+dataset: clic
+dtype: bfloat16
+enabled_test_datasets:
+- clic_edm_qq_pf
+finetune: null
+gpu_batch_multiplier: 256
+gpus: 1
+load: null
+lr: 0.0001
+lr_schedule: cosinedecay
+lr_schedule_config:
+  onecycle:
+    pct_start: 0.3
+make_plots: null
+model:
+  attention:
+    activation: relu
+    attention_type: flash
+    conv_type: attention
+    dropout_conv_id_ff: 0.0
+    dropout_conv_id_mha: 0.0
+    dropout_conv_reg_ff: 0.0
+    dropout_conv_reg_mha: 0.0
+    dropout_ff: 0.0
+    head_dim: 32
+    num_convs: 3
+    num_heads: 32
+    use_pre_layernorm: true
+  cos_phi_mode: linear
+  energy_mode: direct-elemtype-split
+  eta_mode: linear
+  gnn_lsh:
+    activation: elu
+    bin_size: 32
+    conv_type: gnn_lsh
+    distance_dim: 128
+    embedding_dim: 1024
+    ffn_dist_hidden_dim: 128
+    ffn_dist_num_layers: 2
+    layernorm: true
+    max_num_bins: 200
+    num_convs: 3
+    num_node_messages: 2
+    width: 1024
+  input_encoding: split
+  learned_representation_mode: last
+  mamba:
+    activation: elu
+    conv_type: mamba
+    d_conv: 4
+    d_state: 16
+    dropout: 0.0
+    embedding_dim: 128
+    expand: 2
+    num_convs: 2
+    num_heads: 2
+    width: 128
+  pt_mode: direct-elemtype-split
+  sin_phi_mode: linear
+  trainable: all
+ntest: 2000
+ntrain: 4000000
+num_epochs: 10
+num_workers: 8
+nvalid: null
+patience: 20
+prefetch_factor: 100
+raytune:
+  asha:
+    brackets: 1
+    grace_period: 10
+    max_t: 200
+    reduction_factor: 4
+  default_metric: val_loss
+  default_mode: min
+  hyperband:
+    max_t: 200
+    reduction_factor: 4
+  hyperopt:
+    n_random_steps: 10
+  local_dir: null
+  nevergrad:
+    n_random_steps: 10
+  sched: null
+  search_alg: null
+save_attention: true
+sort_data: false
+test: null
+test_dataset:
+  clic_edm_qq_pf:
+    splits:
+    - 1
+    - 2
+    - 3
+    - 4
+    - 5
+    - 6
+    - 7
+    - 8
+    - 9
+    - 10
+    version: 2.5.0
+  clic_edm_ttbar_pf:
+    splits:
+    - 1
+    - 2
+    - 3
+    - 4
+    - 5
+    - 6
+    - 7
+    - 8
+    - 9
+    - 10
+    version: 2.5.0
+  clic_edm_ww_fullhad_pf:
+    splits:
+    - 1
+    - 2
+    - 3
+    - 4
+    - 5
+    - 6
+    - 7
+    - 8
+    - 9
+    - 10
+    version: 2.5.0
+train: true
+train_dataset:
+  clic:
+    physical:
+      batch_size: 1
+      samples:
+        clic_edm_qq_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+        clic_edm_ttbar_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+        clic_edm_ww_fullhad_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+val_freq: null
+valid_dataset:
+  clic:
+    physical:
+      batch_size: 1
+      samples:
+        clic_edm_qq_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+        clic_edm_ttbar_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0
+        clic_edm_ww_fullhad_pf:
+          splits:
+          - 1
+          - 2
+          - 3
+          - 4
+          - 5
+          - 6
+          - 7
+          - 8
+          - 9
+          - 10
+          version: 2.5.0