KVExpSetUpDB.cpp

//Created by KVClassFactory on Fri Jul 20 15:49:04 2018
//Author: eindra
 
#include "KVExpSetUpDB.h"
#include "KVDataSet.h"
#include "KVMultiDetArray.h"
 
#include <TKey.h>
 
ClassImp(KVExpSetUpDB)
 
 
 
KVExpSetUpDB::KVExpSetUpDB()
   : KVExpDB()
{
   // Default constructor
}
 
 
 
 
 
KVExpSetUpDB::KVExpSetUpDB(const Char_t* name)
   : KVExpDB(name)
{
   // Constructor inherited from KVExpDB
}
 
 
 
 
 
KVExpSetUpDB::KVExpSetUpDB(const Char_t* name, const Char_t* title)
   : KVExpDB(name, title)
{
   // Constructor inherited from KVExpDB
}
 
 
 
 
 
KVExpSetUpDB::~KVExpSetUpDB()
{
   // Destructor
}
 
 
 
 
void KVExpSetUpDB::Build(bool minimal)
{
   // \param[in] opt if =true, only a minimal database with runs/systems info is built
   //
   // Build the database.
   // Runs & Systems tables are handled by us, calibrations are handled by each multidetector
   //
   // If ParameterFiles.dat file exists for dataset, we read any files it lists and
   // add the corresponding parameter values to the appropriate KVDBRunFile objects
   //
   // If no run infos are available (perhaps no data yet available for dataset), we make a dummy
   // database with runs numbered from 1 to 100.
 
   try {
      FillRunsTable();
   }
   catch (std::runtime_error& rte) {
 
      // an exception may be thrown if a 'bad' runfile is not found,
      // which indicates an inconsistency in bad_runfiles.dat
      if (TString(rte.what()) != "run infos file not found") throw;
 
      Info("Build", "No informations on dataset runs available");
      Info("Build", "Dummy database runs numbered from 1 to 100 will be created");
      for (int i = 1; i <= 100; ++i) {
         AddRun(new KVDBRun(i, "Dummy database run"));
      }
   }
 
   ReadComments();
   ReadSystemList();
 
   if(minimal)
   {
      // minimal DB has only run/system infos
      minimal_db = true;
      return;
   }
 
   ReadScalerInfos();
   ReadParameterFiles();
 
   KVMultiDetArray::MakeMultiDetector(fDataSet, -1, "KVMultiDetArray", this);
   gMultiDetArray->MakeCalibrationTables(this);
}
 
 
 
 
void KVExpSetUpDB::ReadScalerInfos()
{
   // Look for file scalers.root and read scalers from it
   // scalers are assumed to be stored as 64-bit parameters in the list
 
   TString runinfos = KVDataSet::GetFullPathToDataSetFile(fDataSet, "scalers.root");
   if (runinfos == "") return;
 
   Info("ReadScalerInfos", "Reading scaler infos from %s", runinfos.Data());
   TFile runinfos_file(runinfos);
   TIter it_run(GetRuns());
   KVDBRun* run;
   while ((run = (KVDBRun*)it_run())) {
      auto file_list = run->GetRunIndexList();
      for (auto& rf : file_list) {
         auto scalist = (KVNameValueList*)runinfos_file.Get(Form("run%s", rf.as_string().Data()));
         if (!scalist) scalist = (KVNameValueList*)runinfos_file.Get(Form("run_%06d", run->GetNumber() * index_multiplier));
         if (scalist) {
            int npar = scalist->GetNpar();
            for (int i = 0; i < npar; i += 2) {
               TString parname = scalist->GetParameter(i)->GetName();
               parname.Remove(parname.Length() - 3, 3);
               run->GetRunFile(rf).SetScaler64(parname, scalist->GetValue64bit(parname));
            }
         }
      }
   }
}
 
 
 
 
void KVExpSetUpDB::ReadParameterFiles()
{
   // If a file ParameterFiles.dat is present in the dataset files, we read all files
   // listed in it and add the corresponding parameters to the KVDBRunFile for each runfile.
 
   TString filename = "ParameterFiles.dat";
   if (KVDataSet::FindDataSetFile(fDataSet, filename)) {
      KVFileReader fr;
      auto path = KVDataSet::GetFullPathToDataSetFile(fDataSet, filename);
      Info("ReadParameterFiles", "Using file %s", path.Data());
      fr.OpenFileToRead(path);
      while (fr.IsOK()) {
         fr.ReadLine(0);
 
         if (fr.GetCurrentLine() != "" && !fr.GetCurrentLine().BeginsWith("#"))
            LoadParameters(fr.GetCurrentLine());
      }
      fr.CloseFile();
   }
}
 
 
 
 
void KVExpSetUpDB::LoadParameters(const KVString& filename)
{
   // Read parameter values from filename and attribute them to each KVDBRunFile
   //
   // Format of file:
   //~~~
   //Parameter.Name: [name]
   //Parameter.Type: [double or string]
   //[Parameter.Units: [units]]
   //
   //[runfilelist1]: [value1]
   //[runfilelist2]: [value2]
   //~~~
   //
   // The `Parameter.Units` is optional. If given, an additional parameter,
   // `[name].Units` will be added to each KVDBRunFile.
   //
   // The lists of runfiles use the syntax detailed in KVExpDB::SetRunIndexListFromString().
 
   TString fullpath = "";
   if (!KVBase::SearchKVFile(filename, fullpath, fDataSet)) {
      Info("LoadParameters", "%s does not exist or not found", filename.Data());
      return;
   }
 
   Info("LoadParameters", "file : %s found", fullpath.Data());
   TEnv env;
   env.ReadFile(fullpath, kEnvAll);
 
   KVString parname = env.GetValue("Parameter.Name", "toto");
   KVString partype = env.GetValue("Parameter.Type", "Double_t");
   KVString parunits = env.GetValue("Parameter.Units", "");
 
   partype.ToUpper();
   if (partype.Contains("DOUBLE"))
      partype = "D";
   else
      partype = "S";
 
   TIter next(env.GetTable());
   TEnvRec* rec = 0;
 
   while ((rec = (TEnvRec*)next())) {
 
      KVString sname(rec->GetName());
      if (sname.BeginsWith("Parameter")) continue;
 
      auto runlist = SetRunIndexListFromString(sname);
 
      KVString parval(rec->GetValue());
 
      for (auto& run : runlist) {
         if (partype == "D")
            GetDBRunFile(run).Set(parname, parval.Atof());
         else
            GetDBRunFile(run).Set(parname, parval);
         if (parunits != "")
            GetDBRunFile(run).Set(Form("%s.Units", parname.Data()), parunits);
      }
   }
}
 
 
 
 
void KVExpSetUpDB::FillRunsTable()
{
   // Fill the Runs table using the informations in file runinfos.root
   // (which can be generated using KVRunListCreator).
   //
   // If there are no run infos available (perhaps because no data yet exists for the dataset),
   // this method throws an exception and the database will have a dummy list of runs
   // numbered from 1 to 100
   //
   // Any runfiles which are bad i.e. not to be considered for further analysis should
   // be written in a file bad_runfiles.dat with the given format (TEnv):
   //   - comment lines begin with '#'
   //   - BadFiles.First: first runfile of each run in list is bad
   //   - BadFiles.All: all runfiles for each run in list are bad
   //   - BadFiles.Singles: individual runfiles given as run.index are bad
   //~~~
   //BadFiles.First: 17-26 30 35-42 44 52-54 57-63 65-73 76 83-111 123-141
   //BadFiles.All: 16 27-29 43 46 51 55 56 64 74 75 110 112-122 138
   //BadFiles.Singles: 26.101 86.8 108.21 109.46 111.31 137.9
   //~~~
 
   TString runinfos = KVDataSet::GetFullPathToDataSetFile(fDataSet, "runinfos.root");
 
   if (runinfos == "")
      throw std::runtime_error("run infos file not found");
 
   int run_num{-1}, runfile_index{-1};
 
   Info("FillRunsTable", "Reading run infos from %s", runinfos.Data());
   TFile runinfos_file(runinfos);
   TIter it(runinfos_file.GetListOfKeys());
   TKey* run_key;
   KVList garbage;
   run_index_t run_index;
   while ((run_key = (TKey*)it())) {
      if (TString(run_key->GetClassName()) == "KVNameValueList") {
         // make sure we only use the highest cycle number of each key
         if (run_key->GetCycle() == runinfos_file.GetKey(run_key->GetName())->GetCycle()) {
            KVNameValueList* run = (KVNameValueList*)run_key->ReadObj();
            garbage.Add(run);
            if (with_index_multiplier) {
               run_num = run->GetIntValue("Run") / index_multiplier;
               runfile_index = run->GetIntValue("Run") % index_multiplier;
               if (!runfile_index) run_index = run_index_t{run_num, std::nullopt};
               else run_index = run_index_t{run_num, runfile_index};
            }
            else if (HasRawFilesWithIndex()) {
               run_num = run->GetIntValue("Run");
               KVDBRun* dbrun = GetDBRun(run_num);
               if (!dbrun) {
                  dbrun = new KVDBRun;
                  dbrun->SetNumber(run_num);
                  AddRun(dbrun);
               }
               KVNumberList index_list(run->GetStringValue("IndexList"));
               for (auto index : index_list) {
                  run_index = index ? run_index_t{run_num, index} :
                              run_index_t{run_num, std::nullopt};
                  auto runfile = new KVDBRunFile("", run_index);
                  runfile->SetStartDate(run->GetStringValue(Form("Start.%d", index)));
                  runfile->SetEndDate(run->GetStringValue(Form("End.%d", index)));
                  if (run->HasValue64bit(Form("Size.%d", index)))
                     runfile->SetSize(run->GetValue64bit(Form("Size.%d", index)) / 1024. / 1024.);
                  else
                     runfile->SetSize(run->GetIntValue(Form("Size.%d", index)) / 1024. / 1024.);
 
                  if (run->HasValue64bit(Form("Events.%d", index))) {
                     runfile->SetEvents(run->GetValue64bit(Form("Events.%d", index)));
                  }
                  else
                     runfile->SetEvents(run->GetIntValue(Form("Events.%d", index)));
                  dbrun->AddRunFile(runfile);
               }
               continue;
            }
            else {
               run_num = run->GetIntValue("Run");
               run_index = run_index_t{run_num, std::nullopt};
            }
            KVDBRun* dbrun = GetDBRun(run_num);
            if (!dbrun) {
               dbrun = new KVDBRun;
               dbrun->SetNumber(run_num);
               AddRun(dbrun);
            }
            auto runfile = new KVDBRunFile("", run_index);
            runfile->SetStartDate(run->GetStringValue("Start"));
            runfile->SetEndDate(run->GetStringValue("End"));
            if (run->HasValue64bit("Size"))
               runfile->SetSize(run->GetValue64bit("Size") / 1024. / 1024.);
            else
               runfile->SetSize(run->GetIntValue("Size") / 1024. / 1024.);
 
            if (run->HasValue64bit("Events")) {
               runfile->SetEvents(run->GetValue64bit("Events"));
            }
            else
               runfile->SetEvents(run->GetIntValue("Events"));
            dbrun->AddRunFile(runfile);
         }
      }
   }
   TString badinfos = KVDataSet::GetFullPathToDataSetFile(fDataSet, "bad_runfiles.dat");
   if (!badinfos.IsNull()) {
      Info("FillRunsTable", "Reading bad runfile infos from %s", badinfos.Data());
      TEnv badfiles;
      badfiles.ReadFile(badinfos, kEnvLocal);
      KVNumberList all(badfiles.GetValue("BadFiles.All", ""));
      all.Begin();
      while (!all.End()) {
         auto br = all.Next();
         auto dbrun = GetDBRun(br);
         if (!dbrun) {
            Error("FillRunsTable", "%s:BadFiles.All contains run number %d which is not in %s",
                  badinfos.Data(), br, runinfos.Data());
            throw std::runtime_error(Form("inconsistency between %s:BadFiles.All and %s",
                                          badinfos.Data(), runinfos.Data()));
         }
         dbrun->SetBadAllRunFiles();
      }
      KVNumberList first(badfiles.GetValue("BadFiles.First", ""));
      first.Begin();
      while (!first.End()) {
         auto br = first.Next();
         auto dbrun = GetDBRun(br);
         if (!dbrun) {
            Error("FillRunsTable", "%s:BadFiles.First contains run number %d which is not in %s",
                  badinfos.Data(), br, runinfos.Data());
            throw std::runtime_error(Form("inconsistency between %s:BadFiles.First and %s",
                                          badinfos.Data(), runinfos.Data()));
         }
         dbrun->GetFirstFile().SetBad();
      }
      run_index_list singles(badfiles.GetValue("BadFiles.Singles", ""));
      for (auto& ri : singles) {
         try {
            GetDBRunFile(ri).SetBad();
         }
         catch (std::exception& e) {
            Error("FillRunsTable", "%s:BadFiles.Singles contains run_index %s which is unknown",
                  badinfos.Data(), ri.as_string().Data());
            throw;
         }
      }
   }
   // now set global runfile index for all good files
   Int_t global_runfile_number = 0;
   for (auto run : fListOfRuns) {
      auto r = GetDBRun(run);
      auto ril = r->GetRunIndexList();
      for (auto& ri : ril) {
         if (!GetDBRunFile(ri).IsBad()) {
            GetDBRunFile(ri).SetGlobalRunFileNumber(++global_runfile_number);
            fMapGlobRunFileNumberToRunIndex[global_runfile_number] = ri;
         }
      }
   }
}
 
 
//____________________________________________________________________________//