Fit with theory covmat with n3fit

n3fit/runcards/examples/Basic_feature_scaling.yml

@@ -32,7 +32,9 @@ datacuts:
 ############################################################
 theory:
   theoryid: 200       # database id
-
+sampling:
+  use_t0: false
+  separate_multiplicative: true
 ############################################################
 trvlseed: 1
 nnseed: 2

n3fit/runcards/examples/Basic_hyperopt.yml

@@ -48,6 +48,10 @@ datacuts:
 theory:
   theoryid: 200        # database id
 
+sampling:
+  use_t0: false
+  separate_multiplicative: true
+
 hyperscan_config:
   stopping:
       min_epochs: 1e3

n3fit/runcards/examples/Basic_runcard.yml

@@ -33,6 +33,10 @@ datacuts:
 theory:
   theoryid: 200       # database id
 
+sampling:
+  use_t0: false
+  separate_multiplicative: true
+
 parameters: # This defines the parameter dictionary that is passed to the Model Trainer
   nodes_per_layer: [15, 10, 8]
   activation_per_layer: ['sigmoid', 'sigmoid', 'linear']

n3fit/runcards/examples/Basic_runcard_parallel.yml

@@ -34,6 +34,9 @@ datacuts:
 theory:
   theoryid: 162        # database id
 
+sampling:
+  use_t0: false
+  separate_multiplicative: true
 ############################################################
 trvlseed: 1
 nnseed: 2

n3fit/runcards/examples/DIS_diagonal_l2reg_example.yml

@@ -56,6 +56,10 @@ datacuts:
 theory:
   theoryid: 162        # database id
 
+sampling:
+  use_t0: false
+  separate_multiplicative: true
+
 ############################################################
 trvlseed: 1
 nnseed: 2

n3fit/runcards/examples/Fit_with_theory_covmat.yml

@@ -0,0 +1,137 @@
+#
+# Configuration file for n3fit
+#
+##########################################################################################
+description: "NNPDF4.0 methodology fit with theory covariance matrix (9pt prescription) like NNPDF31_nlo_as_0118_scalecov_9pt"
+
+##########################################################################################
+# frac: training fraction
+# ewk: apply ewk k-factors
+# sys: systematics treatment (see systypes)
+dataset_inputs:
+  - {dataset: NMCPD, frac: 0.5}
+  - {dataset: NMC, frac: 0.5}
+  - {dataset: SLACP, frac: 0.5}
+  - {dataset: SLACD, frac: 0.5}
+  - {dataset: BCDMSP, frac: 0.5}
+  - {dataset: BCDMSD, frac: 0.5}
+  - {dataset: CHORUSNU, frac: 0.5}
+  - {dataset: CHORUSNB, frac: 0.5}
+  - {dataset: NTVNUDMN, frac: 0.5}
+  - {dataset: NTVNBDMN, frac: 0.5}
+  - {dataset: HERACOMBNCEM, frac: 0.5}
+  - {dataset: HERACOMBNCEP460, frac: 0.5}
+  - {dataset: HERACOMBNCEP575, frac: 0.5}
+  - {dataset: HERACOMBNCEP820, frac: 0.5}
+  - {dataset: HERACOMBNCEP920, frac: 0.5}
+  - {dataset: HERACOMBCCEM, frac: 0.5}
+  - {dataset: HERACOMBCCEP, frac: 0.5}
+  - {dataset: HERAF2CHARM, frac: 0.5}
+  - {dataset: CDFZRAP, frac: 1.0}
+  - {dataset: D0ZRAP, frac: 1.0}
+  - {dataset: D0WEASY, frac: 1.0}
+  - {dataset: D0WMASY, frac: 1.0}
+  - {dataset: ATLASWZRAP36PB, frac: 1.0}
+  - {dataset: ATLASZHIGHMASS49FB, frac: 1.0}
+  - {dataset: ATLASLOMASSDY11EXT, frac: 1.0}
+  - {dataset: ATLASWZRAP11, frac: 0.5}
+  - {dataset: ATLAS1JET11, frac: 0.5}
+  - {dataset: ATLASZPT8TEVMDIST, frac: 0.5}
+  - {dataset: ATLASZPT8TEVYDIST, frac: 0.5}
+  - {dataset: ATLASTTBARTOT, frac: 1.0}
+  - {dataset: ATLASTOPDIFF8TEVTRAPNORM, frac: 1.0}
+  - {dataset: CMSWEASY840PB, frac: 1.0}
+  - {dataset: CMSWMASY47FB, frac: 1.0}
+  - {dataset: CMSDY2D11, frac: 0.5}
+  - {dataset: CMSWMU8TEV, frac: 1.0}
+  - {dataset: CMSZDIFF12, frac: 1.0, cfac: [NRM]}
+  - {dataset: CMSJETS11, frac: 0.5}
+  - {dataset: CMSTTBARTOT, frac: 1.0}
+  - {dataset: CMSTOPDIFF8TEVTTRAPNORM, frac: 1.0}
+  - {dataset: LHCBZ940PB, frac: 1.0}
+  - {dataset: LHCBZEE2FB, frac: 1.0}
+  - {dataset: LHCBWZMU7TEV, frac: 1.0, cfac: [NRM]}
+  - {dataset: LHCBWZMU8TEV, frac: 1.0, cfac: [NRM]}
+
+
+############################################################
+datacuts:  
+  t0pdfset: 190310-tg-nlo-global     # PDF set to generate t0 covmat
+  q2min: 13.96                       # Q2 minimum
+  w2min: 12.5                        # W2 minimum
+  use_cuts: fromintersection
+  cuts_intersection_spec:
+  - theoryid: 163
+  - theoryid: 53 
+
+############################################################
+theory:
+  theoryid: 163        # database id
+theorycovmatconfig:
+  point_prescription: "9 point"
+  theoryids:
+    from_: scale_variation_theories
+  pdf: NNPDF31_nlo_as_0118
+  use_thcovmat_in_fitting: true
+  use_thcovmat_in_sampling: true
+
+sampling:
+  use_t0: false
+  separate_multiplicative: false
+
+############################################################
+trvlseed: 376191634
+nnseed: 2080989803
+mcseed: 75955222
+save: false
+genrep: true      # true = generate MC replicas, false = use real data
+
+parameters: # This defines the parameter dictionary that is passed to the Model Trainer
+  nodes_per_layer: [25, 20, 8]
+  activation_per_layer: [tanh, tanh, linear]
+  initializer: glorot_normal
+  optimizer:
+    clipnorm: 6.073e-6
+    learning_rate: 2.621e-3
+    optimizer_name: Nadam
+  epochs: 17000
+  positivity:
+    initial: 184.8
+    multiplier:
+  stopping_patience: 0.1
+  layer_type: dense
+  dropout: 0.0
+  threshold_chi2: 3.5
+############################################################
+fitting:
+  fitbasis: EVOL  # EVOL (7), EVOLQED (8), etc.
+  basis:
+  - {fl: sng, trainable: false, smallx: [1.046, 1.188], largex: [1.437, 2.716]}
+  - {fl: g, trainable: false, smallx: [0.9604, 1.23], largex: [0.08459, 6.137]}
+  - {fl: v, trainable: false, smallx: [0.5656, 0.7242], largex: [1.153, 2.838]}
+  - {fl: v3, trainable: false, smallx: [0.1521, 0.5611], largex: [1.236, 2.976]}
+  - {fl: v8, trainable: false, smallx: [0.5264, 0.7246], largex: [0.6919, 3.198]}
+  - {fl: t3, trainable: false, smallx: [-0.3687, 1.459], largex: [1.664, 3.373]}
+  - {fl: t8, trainable: false, smallx: [0.5357, 1.267], largex: [1.433, 2.866]}
+  - {fl: t15, trainable: false, smallx: [1.073, 1.164], largex: [1.503, 3.636]}
+
+############################################################
+positivity:
+  posdatasets:
+  - {dataset: POSF2U, maxlambda: 1e6}        # Positivity Lagrange Multiplier
+  - {dataset: POSF2DW, maxlambda: 1e6}
+  - {dataset: POSF2S, maxlambda: 1e6}
+  - {dataset: POSFLL, maxlambda: 1e6}
+  - {dataset: POSDYU, maxlambda: 1e10}
+  - {dataset: POSDYD, maxlambda: 1e10}
+  - {dataset: POSDYS, maxlambda: 1e10}
+
+############################################################
+integrability:
+  integdatasets:
+  - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: INTEGXT3, maxlambda: 1e2}
+
+############################################################
+debug: False
+maxcores: 4

n3fit/runcards/examples/developing.yml

@@ -62,6 +62,10 @@ datacuts:
 theory:
   theoryid: 200        # database id
 
+sampling:
+  use_t0: false
+  separate_multiplicative: true
+
 ############################################################
 trvlseed: 1
 nnseed: 2

n3fit/src/n3fit/performfit.py

@@ -37,7 +37,7 @@ def performfit(
     tensorboard=None,
     debug=False,
     maxcores=None,
-    parallel_models=False
+    parallel_models=False,
 ):
     """
         This action will (upon having read a validcard) process a full PDF fit
@@ -148,7 +148,6 @@ def performfit(
     #       [list of all NN seeds]
     #       )
     #
-
     n_models = len(replicas_nnseed_fitting_data_dict)
     if parallel_models and n_models != 1:
         replicas, replica_experiments, nnseeds = zip(*replicas_nnseed_fitting_data_dict)

n3fit/src/n3fit/scripts/n3fit_exec.py

@@ -145,7 +145,20 @@ def from_yaml(cls, o, *args, **kwargs):
             validation_action = namespace + "validation_pseudodata"
 
             N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action))
-
+        #Theorycovmat flags and defaults
+        N3FIT_FIXED_CONFIG['theory_covmat_flag'] = False
+        if (thconfig:=file_content.get('theorycovmatconfig')) is not None:
+            N3FIT_FIXED_CONFIG['theory_covmat_flag'] = True
+            N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = thconfig.get('use_thcovmat_in_fitting', True) 
+            N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = thconfig.get('use_thcovmat_in_sampling', True)  
+            N3FIT_FIXED_CONFIG['use_user_uncertainties'] = thconfig.get('use_user_uncertainties', False) 
+            N3FIT_FIXED_CONFIG['use_scalevar_uncertainties'] = thconfig.get('use_scalevar_uncertainties', True) 
+        #Sampling flags
+        if (sam_t0:=file_content.get('sampling')) is not None:
+            N3FIT_FIXED_CONFIG['use_t0_sampling'] = sam_t0.get('use_t0', False) 
+            N3FIT_FIXED_CONFIG['separate_multiplicative'] = sam_t0.get('separate_multiplicative', True) 
+        #Fitting flag
+        N3FIT_FIXED_CONFIG['use_t0_fitting'] = file_content.get('fitting').get('use_t0', True)
         file_content.update(N3FIT_FIXED_CONFIG)
         return cls(file_content, *args, **kwargs)
 

n3fit/src/n3fit/scripts/vp_setupfit.py

@@ -153,9 +153,7 @@ def from_yaml(cls, o, *args, **kwargs):
         else:
             filter_action = 'datacuts::theory::fitting filter'
             check_n3fit_action = 'datacuts::theory::fitting n3fit_checks_action'
-
         SETUPFIT_FIXED_CONFIG['actions_'] += [check_n3fit_action, filter_action]
-
         if file_content.get('theorycovmatconfig') is not None:
             SETUPFIT_FIXED_CONFIG['actions_'].append(
                 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat')

n3fit/src/n3fit/tests/regressions/quickcard_1.json

@@ -20,8 +20,8 @@
     },
     {
       "fl": "v3",
-      "smallx": 0.3031329810619354,
-      "largex": 2.6214101314544678,
+      "smallx": 0.30309754610061646,
+      "largex": 2.6213722229003906,
       "trainable": true
     },
     {
@@ -51,45 +51,45 @@
   ],
   "stop_epoch": 1000,
   "best_epoch": 999,
-  "erf_tr": 36.54704284667969,
-  "erf_vl": 30.701587677001953,
-  "chi2": 22.70619773864746,
+  "erf_tr": 38.022315979003906,
+  "erf_vl": 31.481945037841797,
+  "chi2": 22.705814361572266,
   "pos_state": "POS_VETO",
   "arc_lengths": [
-    1.1037201268771095,
-    1.505700648681965,
-    1.2027295354983405,
-    1.4316397494895012,
-    1.1901164504449115
+    1.103731936457393,
+    1.505681943929697,
+    1.2027105035135508,
+    1.426451562695196,
+    1.1901165080869252
   ],
   "integrability": [
-    0.002630541814142684,
-    0.0026305418141405745,
-    0.00014138085179959958,
-    3.6656303405761737,
-    0.004028238385220817
+    0.002630652510560605,
+    0.0026306525105581624,
+    0.00013975353567441395,
+    3.6658432483673087,
+    0.004028247582030353
   ],
   "timing": {
     "walltime": {
-      "Total": 20.638618230819702,
+      "Total": 12.020201921463013,
       "start": 0.0,
-      "replica_set": 0.0033304691314697266,
-      "replica_fitted": 20.638471841812134,
-      "replica_set_to_replica_fitted": 20.635141372680664
+      "replica_set": 0.003970146179199219,
+      "replica_fitted": 12.02015495300293,
+      "replica_set_to_replica_fitted": 12.01618480682373
     },
     "cputime": {
-      "Total": 27.691918045999998,
+      "Total": 13.138117000000001,
       "start": 0.0,
-      "replica_set": 0.003325130999999537,
-      "replica_fitted": 27.691770436,
-      "replica_set_to_replica_fitted": 27.688445305000002
+      "replica_set": 0.029436999999999713,
+      "replica_fitted": 13.138062999999999,
+      "replica_set_to_replica_fitted": 13.108626
     }
   },
   "version": {
-    "keras": "2.8.0",
-    "tensorflow": "2.8.0, mkl=True",
-    "numpy": "1.22.2",
-    "nnpdf": "4.0.4.152+g84203534c",
-    "validphys": "4.0.4.152+g84203534c"
+    "keras": "2.4.0",
+    "tensorflow": "2.4.1, mkl=False",
+    "numpy": "1.21.2",
+    "nnpdf": "4.0.4.221+gc94f7414b-dev",
+    "validphys": "4.0.4.221+gc94f7414b-dev"
   }
 }