nn trained

main.py

@@ -35,6 +35,13 @@ parser.add_argument(
     default=True,
     help="Enables determination of weights used by neural networks.",
 )
+parser.add_argument(
+    "-p",
+    "--prepare",
+    action="store_true",
+    default=True,
+    help="Enables preparation of data for matching.",
+)
 parser.add_argument(
     "--prepare-params-data",
     action="store_true",
@@ -127,10 +134,12 @@ if args.forward_weights:
 if args.matching_weights:
     os.chdir(os.path.dirname(os.path.realpath(__file__)))
     train_matching_ghost_mlp(
-        prepare_data=True,  # args.prepare_weights_data,
-        input_file="data/ghost_data_B.root",
-        tree_name="PrMatchNN_e6feac0d.PrMCDebugMatchToolNN/MVAInputAndOutput",
+        prepare_data=False,
+        input_file="data/ghost_data_B_electron_weights.root",
+        tree_name="PrMatchNN_3e224c41.PrMCDebugMatchToolNN/MVAInputAndOutput",
         outdir="neural_net_training",
+        exclude_electrons=False,
+        only_electrons=True,
     )
     # this ensures that the directory is correct
     os.chdir(os.path.dirname(os.path.realpath(__file__)))

moore_options/residual_get_ghost_data.py

@@ -4,7 +4,7 @@
 NOT IMPLEMENTED YET
 
 
-Moore/run gaudirun.py /work/cetin/LHCb/reco_tuner/moore_options/Recent_get_ghost_data.py
+Moore/run gaudirun.py /work/cetin/LHCb/reco_tuner/moore_options/residual_get_ghost_data.py
 """
 
 from Moore import options, run_reconstruction
@@ -25,7 +25,7 @@ import glob
 
 options.evt_max = -1
 
-decay = "test"  # D, B
+decay = "B"  # D, B
 options.ntuple_file = f"data/ghost_data_{decay}.root"
 options.input_type = "ROOT"
 
@@ -117,6 +117,7 @@ def run_tracking_debug():
         MaxDistX=500,
         MaxDistY=500,
         MaxDSlope=1.5,
+        MinMatchNN=0.45,  # NN response cut value
     )
 
     match_debug = PrMatchNN(

nn_neural_net_training/result/matching.hpp

@@ -0,0 +1,48 @@
+const auto fMin = std::array<simd::float_v, 6>{
+    {1.32643926918e-05, 1.20999777664e-06, 3.81469726562e-06, 1.52587890625e-05,
+     2.20164656639e-06, 1.86264514923e-09}};
+const auto fMax = std::array<simd::float_v, 6>{{14.9999952316, 0.436187297106,
+                                                249.999572754, 399.485595703,
+                                                1.30260443687, 0.148344695568}};
+const auto fWeightMatrix0to1 = std::array<std::array<simd::float_v, 7>, 8>{
+    {{2.32568146034949, -3.97864517484141, -0.976136452226726, 1.84234344676559,
+      -3.10046463102268, 4.13961872392198, 1.32395215581256},
+     {-0.246260592363558, -16.6289365646957, 15.8745926520597, 5.54227150397204,
+      -3.52013322130382, 3.54800430147538, 4.65963029843042},
+     {-0.0480865527472585, -0.629210074395733, 6.00348546361291,
+      2.9051880336304, -0.14352194426084, 1.69533803008533, 8.43612131346998},
+     {0.586453583994425, -2.56124202576808, 2.59227690708752,
+      0.0874243316906918, -2.97381765628525, 5.49796401976845,
+      3.23192359468339},
+     {0.429663439996412, -22.1383805768484, -0.392774946210208,
+      -3.3393241414433, -0.0183236766918373, 1.7443084621404,
+      -23.1241106528584},
+     {1.51561003857451, -0.252437187813493, 3.4382652179148, 1.64873635165153,
+      1.3257641118939, -1.3769915299618, 6.284788658685},
+     {-0.79066972900182, -0.617757099680603, 0.740878002718091,
+      0.681870030239224, -1.20759406685829, 0.769290467724204,
+      -1.8437808630988},
+     {1.26283446391613, 1.060406101318, 0.30016156694275, 0.868137090713936,
+      0.620452727287864, 0.654572151525178, -1.93868171775984}}};
+const auto fWeightMatrix1to2 = std::array<std::array<simd::float_v, 9>, 6>{
+    {{-0.756398914721592, 1.43176897679079, -1.9761225512629,
+      -0.252826703054453, 5.76338466721064, 0.853447490406625, 1.63438201788813,
+      -1.30124222851611, -1.16516476663684},
+     {1.33354118308893, 2.2779204457711, -2.4183940976708, -1.41409141050929,
+      -3.03014280476042, -0.105294409656274, -1.61531096417457,
+      0.0713464687805576, -4.46730787742624},
+     {1.69117951310622, 0.478803367417533, -0.0952992998738417,
+      -1.42291620159966, -5.3475695755735, -0.851706256912453,
+      -0.825543426908553, -1.84634786630319, 1.10300947885605},
+     {1.62294844942986, -1.4305887420849, 1.34690035656602, -1.75196364787073,
+      -1.34911857298729, -1.19784919878849, 1.61348068527877, -1.6413641883722,
+      -1.80987544922642},
+     {-0.885340378859963, -1.27010625003553, 1.64729145944323,
+      -1.93179670311711, -2.00487598846412, 0.858689001379895,
+      -0.848898627795279, 0.783837335125351, -1.50563595386066},
+     {-0.643070342091735, -1.362074820856, 3.23003893144526, -1.8069989021131,
+      -1.52168986931666, -2.92720177768097, -1.52203810494393, 1.54153084775635,
+      4.02998353429178}}};
+const auto fWeightMatrix2to3 = std::array<simd::float_v, 7>{
+    {-1.03488783417574, 0.540010884713827, -1.17870273673375, 1.01943381348885,
+     -0.679190259548567, 1.25798110915057, 2.3197360649145}};

nn_neural_net_training/result_B/matching.hpp

@@ -0,0 +1,46 @@
+const auto fMin = std::array<simd::float_v, 6>{
+    {1.37092808927e-06, 1.07555365503e-06, 0, 1.90734863281e-06,
+     1.73929147422e-05, 1.86264514923e-09}};
+const auto fMax = std::array<simd::float_v, 6>{{14.9999952316, 0.456682443619,
+                                                249.999572754, 399.509643555,
+                                                1.33115208149, 0.149789437652}};
+const auto fWeightMatrix0to1 = std::array<std::array<simd::float_v, 7>, 8>{
+    {{-1.3734781925797, 13.4202721220084, -5.84482699847354, 0.208720202271194,
+      3.52940201568696, -5.35007508017961, 6.10232623582908},
+     {0.269463828190076, 12.2029002280153, 6.20803317501961, -9.43442815316897,
+      2.5338939027162, 5.99544654330182, 16.266514230858},
+     {-0.165852817298963, -12.5570036498389, 19.5108101030614, 10.1445756810778,
+      -4.70591905221782, -9.82613113151628, 2.66946232799658},
+     {0.280264112609391, -40.4573608414915, 4.50829859766595, -9.38270110978156,
+      2.13898954875748, 4.73797410702965, -38.2552994749474},
+     {-15.3913555770922, 1.18454625888548, 1.03308239102009, 2.80096921737441,
+      -1.86435943580432, -5.12259817922783, -14.7182721956392},
+     {-0.473433045504226, 14.9901069695702, -0.236384720797966,
+      -2.83841297397374, 4.98474416815065, -6.59501221410077, 6.97717117093051},
+     {-0.79066972900182, -0.617757099680603, 0.740878002718091,
+      0.681870030239224, -1.20759406685829, 0.769290467724204,
+      -1.8437808630988},
+     {0.142197307909266, 4.84602282950846, -9.65725300640334, 5.68314089024306,
+      0.631054662487241, 0.766483060165155, 2.3260315163825}}};
+const auto fWeightMatrix1to2 = std::array<std::array<simd::float_v, 9>, 6>{
+    {{0.647996552227704, -3.612673407752, 0.218049700051821, 4.89119034256858,
+      -0.00710530398728626, -0.739119819896367, 1.63438201788813,
+      0.7192739388343, -4.39806909742125},
+     {-0.719597437431301, -3.27873531826254, -2.03233412174408,
+      -3.60079441122056, 0.0930923625129556, -2.47473692076248,
+      -1.61531096417457, -1.73667807655155, 3.65065717704823},
+     {2.15115292231327, 0.537266754158749, -0.529575619029901,
+      -0.840914255611436, 1.02786405393109, -2.2383981589872,
+      -0.825543426908553, -0.685116658617715, -1.95672133400954},
+     {0.164139216021613, -0.378666175423714, -1.43567813416239,
+      -1.86509513117207, -0.825083002191541, -1.70460785835385,
+      1.61348068527877, -1.66550797875971, -0.956253568725315},
+     {-1.87493924816154, -0.453672605669931, 0.283493943583684,
+      0.878365550455799, 0.284631862858431, 0.933935190438462,
+      -0.848898627795279, 0.121615867119966, 2.40557433526087},
+     {0.853517633026983, -0.322377109742158, 0.30359642229039,
+      -2.70050427549895, 0.434398564771274, -1.07531792256432,
+      -1.52203810494393, 0.471135339353818, -7.51274733403613}}};
+const auto fWeightMatrix2to3 = std::array<simd::float_v, 7>{
+    {-0.773202850704438, 0.952227138510482, 0.74769506152075, 0.306824902699197,
+     -0.557424643818581, 1.36609661342348, -1.24818793392955}};

nn_neural_net_training/result_B_old/matching.hpp

@@ -0,0 +1,48 @@
+const auto fMin = std::array<simd::float_v, 6>{
+    {1.37092808927e-06, 1.07555365503e-06, 0, 1.90734863281e-06,
+     1.73929147422e-05, 1.86264514923e-09}};
+const auto fMax = std::array<simd::float_v, 6>{{14.9999952316, 0.456682443619,
+                                                249.999572754, 399.509643555,
+                                                1.33115208149, 0.149789437652}};
+const auto fWeightMatrix0to1 = std::array<std::array<simd::float_v, 7>, 8>{
+    {{0.55218535628556, -9.3289553119363, -3.16480805777192, 9.21929582222451,
+      -5.84675321729571, 4.37995011218691, -2.12651852927708},
+     {2.19402229437066, -36.4572143799157, 4.72612050852174, 0.871774263011679,
+      0.308249736812244, 5.59902946146285, -21.3121523564936},
+     {0.326882064023056, 2.35866196875568, 9.48783066071353, 2.75913715527822,
+      -3.60778259684168, 2.80447887380193, 12.22677213297},
+     {0.555959841347612, -11.3379921223552, 24.99514413087, 4.38044026679039,
+      -4.79766508655656, -5.51874542469878, 8.39926399588362},
+     {-0.474573814356478, -45.048645069346, -1.91571008337192,
+      -2.97043145049536, -0.791922976045819, 2.80933052961339,
+      -45.2686657256446},
+     {1.02111090620048, 0.942295739720341, 4.23884295504771, 3.69611210680021,
+      3.06108184531354, -5.59083664638509, 5.48212218750871},
+     {-0.79066972900182, -0.617757099680603, 0.740878002718091,
+      0.681870030239224, -1.20759406685829, 0.769290467724204,
+      -1.8437808630988},
+     {1.25219270646431, 0.549228434890616, 0.470255515433846, 0.916142200504342,
+      1.60846971174291, 0.516066034145183, -1.99907858325808}}};
+const auto fWeightMatrix1to2 = std::array<std::array<simd::float_v, 9>, 6>{
+    {{-2.16740050633671, 1.64201098704318, -1.81457731661729, 0.276267162453127,
+      4.41723045721244, 0.116946763347361, 1.63438201788813, -1.34454525041306,
+      -11.6363132267585},
+     {-0.975733315897721, -0.74456197080548, 1.37299729852781,
+      -0.935058973429512, 0.0844226992748141, -0.132452262552727,
+      -1.61531096417457, -0.186263378023113, 5.02662780750337},
+     {1.04696354000933, 0.278924511733321, -1.35925413801625, 0.938772342837744,
+      -0.549530917541879, -0.520171806146222, -0.825543426908553,
+      -2.06608637235381, -0.791984902945839},
+     {-1.2045961477844, -0.991003979261367, 1.09783625990238,
+      -0.421872249827208, -0.889785288418292, 2.04952712400642,
+      1.61348068527877, -1.7061481912452, -4.6379237728574},
+     {-1.36108475234833, -0.998277929718627, 1.44485269371602,
+      -0.712692589749601, 2.24954768341439, 2.14013866962467,
+      -0.848898627795279, 0.868380765164237, -2.78040856790563},
+     {-0.388348743847599, -3.23828818784509, -3.09515929145523,
+      -1.60979064312646, 2.55518501696684, -2.40442392560053, -1.52203810494393,
+      1.61704406536505, 1.28981466057697}}};
+const auto fWeightMatrix2to3 = std::array<simd::float_v, 7>{
+    {-0.662286199846436, 0.602757344813461, -0.498657128878293,
+     0.682053959836921, -0.846606195204036, 0.885206167679193,
+     -0.091536072257332}};

nn_neural_net_training/result_D/matching.hpp

@@ -0,0 +1,49 @@
+const auto fMin = std::array<simd::float_v, 6>{
+    {8.165359759e-06, 1.20664617498e-06, 3.0517578125e-05, 0, 4.7143548727e-06,
+     5.58793544769e-09}};
+const auto fMax =
+    std::array<simd::float_v, 6>{{14.9999341965, 0.441820472479, 249.991241455,
+                                  399.657226562, 1.31253051758, 0.1461160779}};
+const auto fWeightMatrix0to1 = std::array<std::array<simd::float_v, 7>, 8>{
+    {{-2.69517659211572, 11.8302794023495, -4.18951579686807, -3.98494892798567,
+      2.81897548445767, 0.59383239448013, 8.23409922740496},
+     {0.211366021230384, -17.963369064596, 15.9757126175754, 7.06797978526591,
+      -4.70452883659984, -6.9670945574808, -6.09966951812501},
+     {-0.671572194549583, 11.3044506689324, 0.41567016443692, -1.37717944379749,
+      4.32454960210643, -2.81417446537734, 9.27800394526066},
+     {-0.0170007006326477, -29.3978844207289, 1.21375106319138,
+      -4.08361109078602, 1.26964946956945, 2.36059581879151, -28.6616649803861},
+     {-11.5040478504233, 0.787126057627091, -1.9688816880041, 3.80563620582515,
+      -1.24505398457039, -4.63206817893295, -13.6204407803068},
+     {-0.338909805576579, 5.40829054574145, -5.80255047095045,
+      -4.01690019633219, 1.01720190260241, -8.00726918670078,
+      -9.13220942993612},
+     {-0.79066972900182, -0.617757099680603, 0.740878002718091,
+      0.681870030239224, -1.20759406685829, 0.769290467724204,
+      -1.8437808630988},
+     {-0.0200186919403349, 1.41949954504535, 1.49019129872922,
+      0.288411192617344, -1.04637027529446, 0.461207091311545,
+      2.34712624673865}}};
+const auto fWeightMatrix1to2 = std::array<std::array<simd::float_v, 9>, 6>{
+    {{-0.742932789484951, 1.098742538125, -0.406409364576387, 3.47055509094897,
+      0.0962780863393642, 1.41748292133237, 1.63438201788813, -1.44301381179313,
+      -0.572613401802679},
+     {-0.38589120983735, 1.59861062444015, -0.0248567208616739,
+      0.671741015980856, -0.708380620370054, -1.03895600322296,
+      -1.61531096417457, -0.148523097987218, -4.64632456422582},
+     {0.79166633002489, -1.08475628425482, -4.28859285488566, 1.52323344063281,
+      0.841577416846386, -2.87987947235168, -0.825543426908553,
+      -1.68433960913801, 3.44474663480542},
+     {0.0775004589408732, -0.262461293729405, -1.52083397977799,
+      -1.8717755745741, -0.836405509817299, -1.7218693116007, 1.61348068527877,
+      -1.66550797875971, -0.970612266783855},
+     {-0.173976577204694, 0.622518962366594, 1.06846030554012,
+      -1.98774771637332, 0.519455930696643, 0.29715629978414,
+      -0.848898627795279, -0.571811756436865, -0.634485828880002},
+     {1.01806297385566, -2.23322855713652, -0.6087066354355, -2.48675705217909,
+      3.17812971554116, 0.101672334443862, -1.52203810494393, 2.31992216900119,
+      -1.25181073559493}}};
+const auto fWeightMatrix2to3 = std::array<simd::float_v, 7>{
+    {-0.916964821952665, 0.719312774569769, -0.639131582384414,
+     0.543723763328418, -0.519810071051254, 0.818949275577508,
+     -0.217502220186121}};

nn_neural_net_training/result_D_old/matching.hpp

@@ -0,0 +1,47 @@
+const auto fMin = std::array<simd::float_v, 6>{
+    {3.08839298668e-06, 1.0285064036e-06, 1.52587890625e-05, 0,
+     5.87664544582e-07, 1.16415321827e-10}};
+const auto fMax =
+    std::array<simd::float_v, 6>{{14.9999723434, 0.448565632105, 249.991241455,
+                                  399.657226562, 1.32571601868, 0.1461160779}};
+const auto fWeightMatrix0to1 = std::array<std::array<simd::float_v, 7>, 8>{
+    {{-13.6018653076529, 11.5780217700141, -7.92762809494091, -2.3767990231665,
+      2.10509041357149, 8.93423542038951, 0.697736541430846},
+     {1.39148569147387, -18.5749654585149, 16.332262515645, 8.93683318362009,
+      -5.31296543840869, -5.3403427435078, -2.19396356951465},
+     {-1.01323411158617, 13.2753123794943, 0.728991860392637, -2.42297786296918,
+      5.31377513515812, -3.50060317341991, 10.417424252956},
+     {-0.248243535822069, 4.62216903283789, 7.02215266119243, 1.16722623835237,
+      -4.02343144066426, 0.795833957766165, 8.68951250524976},
+     {-0.238717750484162, 6.4095254209171, -7.18004762765776, -5.26488261250603,
+      0.399079753011244, -13.2043917021304, -15.6484370000787},
+     {0.28927080766293, -43.0775712799999, 1.66954473021466, -9.33896425089968,
+      2.33665742943925, 3.79800824384931, -44.3378970188981},
+     {-0.79066972900182, -0.617757099680603, 0.740878002718091,
+      0.681870030239224, -1.20759406685829, 0.769290467724204,
+      -1.8437808630988},
+     {1.40243557561751, 0.527362898119982, 0.45726589950568, 1.14682278333905,
+      1.07970493015474, -0.120090795589863, -1.93859670804163}}};
+const auto fWeightMatrix1to2 = std::array<std::array<simd::float_v, 9>, 6>{
+    {{-0.799170659507791, 0.78794128149515, -0.763826599227941,
+      -2.3771947370175, 1.02090569194105, 2.93661596670106, 1.63438201788813,
+      -1.4315640726598, -1.65256239855233},
+     {-0.0840828763430264, 1.63030483445294, 0.480480602063334,
+      -2.6196066367932, -1.07206902633681, 1.70077768270329, -1.61531096417457,
+      0.0827459973313509, -6.82577663153282},
+     {0.549379141222342, -1.30994855822444, -3.47047538273556,
+      0.416631880451092, -2.01641324755852, 0.534999953845232,
+      -0.825543426908553, -1.89592023892521, 5.51877157805828},
+     {0.0804714249535426, -0.5308079142129, -1.48689873935011,
+      -1.86763554052357, -0.869089360209786, -1.67763600182079,
+      1.61348068527877, -1.66550797875971, -0.925481963732789},
+     {-0.686375033428724, 1.09398610198181, 0.699349709460149,
+      -1.04209787556848, 0.0477294646540392, -0.311194459626976,
+      -0.848898627795279, 1.21798575421877, -1.20136465619996},
+     {0.65672978185887, -2.41522086895727, -0.906588505776888, 1.17488116346046,
+      0.348225140957002, -1.76790548692959, -1.52203810494393, 1.20010038210504,
+      2.16681827421459}}};
+const auto fWeightMatrix2to3 = std::array<simd::float_v, 7>{
+    {-0.711664725241253, 0.506164178116774, -0.741743336419543,
+     0.501270635463003, -0.672368683770616, 0.747306441658917,
+     0.789949973283111}};

parameterisations/train_matching_ghost_mlps.py

@@ -10,10 +10,11 @@ def train_matching_ghost_mlp(
     input_file: str = "data/ghost_data_B.root",
     tree_name: str = "PrMatchNN.PrMCDebugMatchToolNN/MVAInputAndOutput",
     exclude_electrons: bool = True,
-    n_train_signal: int = 50e3,
-    n_train_bkg: int = 500e3,
-    n_test_signal: int = 10e3,
-    n_test_bkg: int = 100e3,
+    only_electrons: bool = False,
+    n_train_signal: int = 100e3,
+    n_train_bkg: int = 100e3,
+    n_test_signal: int = 50e3,
+    n_test_bkg: int = 50e3,
     prepare_data: bool = True,
     outdir: str = "neural_net_training",
 ):
@@ -33,16 +34,22 @@ def train_matching_ghost_mlp(
         rdf = ROOT.RDataFrame(tree_name, input_file)
         if exclude_electrons:
             rdf_signal = rdf.Filter(
-                "quality == 1",  # -1 elec, 0 ghost, 1 all part wo elec
+                "mc_quality == 1",  # -1 elec, 0 ghost, 1 all part wo elec
                 "Signal is defined as one label (excluding electrons)",
             )
-            rdf_bkg = rdf.Filter("quality == 0", "Ghosts are defined as zero label")
+            rdf_bkg = rdf.Filter("mc_quality == 0", "Ghosts are defined as zero label")
+        elif only_electrons:
+            rdf_signal = rdf.Filter(
+                "mc_quality == -1",  # -1 elec, 0 ghost, 1 all part wo elec
+                "Signal is defined as one label (excluding electrons)",
+            )
+            rdf_bkg = rdf.Filter("mc_quality == 0", "Ghosts are defined as zero label")
         else:
             rdf_signal = rdf.Filter(
-                "abs(quality) > 0",
+                "abs(mc_quality) > 0",
                 "Signal is defined as non-zero label",
             )
-            rdf_bkg = rdf.Filter("quality == 0", "Ghosts are defined as zero label")
+            rdf_bkg = rdf.Filter("mc_quality == 0", "Ghosts are defined as zero label")
         rdf_signal.Snapshot(
             "Signal",
             outdir + "/" + input_file.strip(".root") + "_matching_signal.root",
@@ -75,12 +82,12 @@ def train_matching_ghost_mlp(
     factory.SetVerbose(True)
     dataloader = TMVA.DataLoader("MatchNNDataSet")
 
-    dataloader.AddVariable("chi2", "F")
-    dataloader.AddVariable("teta2", "F")
-    dataloader.AddVariable("distX", "F")
-    dataloader.AddVariable("distY", "F")
-    dataloader.AddVariable("dSlope", "F")
-    dataloader.AddVariable("dSlopeY", "F")
+    dataloader.AddVariable("mc_chi2", "F")
+    dataloader.AddVariable("mc_teta2", "F")
+    dataloader.AddVariable("mc_distX", "F")
+    dataloader.AddVariable("mc_distY", "F")
+    dataloader.AddVariable("mc_dSlope", "F")
+    dataloader.AddVariable("mc_dSlopeY", "F")
     # dataloader.AddVariable("tx", "F")
     # dataloader.AddVariable("ty", "F")
     # dataloader.AddVariable("tx_scifi", "F")
@@ -91,7 +98,7 @@ def train_matching_ghost_mlp(
 
     # these cuts are also applied in the algorithm
     preselectionCuts = ROOT.TCut(
-        "chi2<15 && distX<250 && distY<400 && dSlope<1.5 && dSlopeY<0.15",  #### ganz raus für elektronen
+        "mc_chi2<30 && mc_distX<500 && mc_distY<500 && mc_dSlope<3.0 && mc_dSlopeY<0.3",  #### ganz raus für elektronen
     )
     dataloader.PrepareTrainingAndTestTree(
         preselectionCuts,