KVSimDataManager.cpp

#include "KVSimDataManager.h"
#include "TSystem.h"
#include "KVSimDir.h"
#include "KVSimFile.h"
#include "KVDataSetManager.h"
 
ClassImp(KVSimDataManager)
 
 
 
void KVSimDataManager::OpenDataBase(const TString& filepath)
{
   // Open/create the SQLite database file with given path
   //
   // Creates default table(s) if not already done
 
   fDB.open(filepath);
 
   if (!fDB.has_table("datasets")) {
      {
         KVSQLite::table T("models");
         T.add_column("id", "INTEGER");
         T.add_column("name", "TEXT").NOT_NULL();
         T.primary_key("id");
         fDB.add_table(T);
      }
      {
         KVSQLite::table T("datasets");
         T.add_column("id", "INTEGER");
         T.add_column("user", "TEXT").NOT_NULL();
         T.add_column("description", "TEXT").NOT_NULL();
         T.add_column("model_id", "INTEGER");
         T.primary_key("id");
         T.foreign_key("model_id", "models", "id");
         fDB.add_table(T);
      }
      {
         KVSQLite::table T("reactions");
         T.add_column("id", "INTEGER");
         T.add_column("nuc1", "TEXT").NOT_NULL(); // projectile if nuc1 & nuc2 defined, else compound nucleus
         T.add_column("nuc2", "TEXT");
         T.add_column("energy", "REAL"); // either beam energy in MeV for reactions, or excitation energy of CN in MeV
         T.add_column("spin_hbar", "INTEGER"); // spin of CN, if fixed value used
         T.add_column("impact_parameter", "REAL"); // fixed value of impact parameter, if used
         T.add_column("ip_units", "TEXT"); // impact parameter units, if used: "fm" or "rel" (=> b/bmax)
         T.primary_key("id");
         fDB.add_table(T);
      }
      {
         KVSQLite::table T("repositories");
         T.add_column("id", "INTEGER");
         T.primary_key("id");
         T.add_column("root_dir", "TEXT").NOT_NULL();
         T.add_column("type", "TEXT").NOT_NULL().DEFAULT("local");
         T.add_column("Xrdsrv", "TEXT");
         T.add_column("Xrdroot", "TEXT");
         fDB.add_table(T);
      }
      {
         KVSQLite::table T("unfiltered_files");
         T.add_column("id", "INTEGER");
         T.add_column("name", "TEXT").NOT_NULL();
         T.add_column("path", "TEXT").NOT_NULL();
         T.add_column("repository_id", "INTEGER"); // may contain id of a repository in table, or NULL
         T.add_column("original_name", "TEXT"); // original name of file if it has been imported to the repository, or NULL
         T.add_column("original_path", "TEXT"); // original full path to file if it has been imported to the repository, or NULL
         T.add_column("events", "INTEGER");
         T.add_column("type", "TEXT").NOT_NULL(); // = "primary" or "secondary" i.e. before or after secondary decay
         T.add_column("dataset_id", "INTEGER");
         T.add_column("reaction_id", "INTEGER");
         T.primary_key("id");
         T.foreign_key("dataset_id", "datasets", "id"); // link to dataset
         T.foreign_key("reaction_id", "reactions", "id"); // link to reaction
         fDB.add_table(T);
      }
      {
         KVSQLite::table T("filters"); // table with all filtering parameters
         T.add_column("id", "INTEGER");
         T.add_column("kinematics", "INTEGER");
         T.foreign_key("kinematics", "reactions", "id"); // definition of kinematics used, e.g. lab->CM transformation
         T.add_column("geometry", "TEXT").NOT_NULL(); // experimental dataset used to define geometry of multidetector
         T.add_column("run_number", "INTEGER"); // specific run number used to define experimental conditions, if used
         T.add_column("filter_type", "TEXT").NOT_NULL(); // type of filter = "Geo" or "GeoThresh"
         T.add_column("data_quality_audit", "TEXT"); // name of data quality audit defining identification performances, if used
         T.add_column("random_phi", "TEXT"); // ="yes" if phi randomized for each event, NULL otherwise
         T.add_column("gemini", "INTEGER"); // if KaliVeda Gemini++ used to perform secondary decay just before filter, number of decays per primary event, or NULL
         T.add_column("gem_add_erot", "TEXT"); // if KaliVeda Gemini++ used, "yes" if rotational (Yrast) energy added to E*, or NULL
         T.primary_key("id");
         fDB.add_table(T);
      }
      {
         KVSQLite::table T("filtered_files"); // filtered files
         T.add_column("id", "INTEGER");
         T.add_column("name", "TEXT").NOT_NULL();
         T.add_column("path", "TEXT").NOT_NULL();
         T.add_column("repository_id", "INTEGER"); // may contain id of a repository in table, or NULL
         T.add_column("original_name", "TEXT"); // original name of file if it has been imported to the repository, or NULL
         T.add_column("original_path", "TEXT"); // original full path to file if it has been imported to the repository, or NULL
         T.add_column("events", "INTEGER");
         T.add_column("unfiltered_file_id", "INTEGER"); // link to simulated file processed to produce this one
         T.add_column("dataset_id", "INTEGER");
         T.add_column("reaction_id", "INTEGER");
         T.add_column("filter_id", "INTEGER");
         T.primary_key("id");
         T.foreign_key("unfiltered_file_id", "unfiltered_files", "id");
         T.foreign_key("dataset_id", "datasets", "id"); // link to dataset
         T.foreign_key("reaction_id", "reactions", "id"); // link to reaction
         T.foreign_key("filter_id", "filters", "id"); // link to filter parameters
         fDB.add_table(T);
      }
   }
}
 
 
 
 
KVSimDataManager::filter_id KVSimDataManager::AddFilter(reaction_id kinematics, const TString& geometry, const TString& filter_type,
      bool random_phi, std::optional<int> run_number, std::optional<TString> data_quality_audit, std::optional<int> gem_decay_per_event,
      std::optional<bool> gem_add_rot_energy)
{
   // Add a set of filtering conditions to the database and return the corresponding index
   //
   // \param[in] kinematics index of reaction used to define lab->CM transform
   // \param[in] geometry name of experimental dataset used to define multidetector array geometry
   // \param[in] filter_type can be "Geo" or "GeoThresh" (or "Full" for old filtered simulations before KaliVeda v1.15)
   // \param[in] random_phi [default=true] whether each event underwent a random azimuthal rotation before filtering
   // \param[in] run_number [optional] specific run number of dataset used to define experimental conditions
   // \param[in] data_quality_audit [optional] name of audit used to define identification performances
   // \param[in] gem_decay_per_event [optional] if Gemini++ used to perform secondary decay just before filtering, number of decays per primary event
   // \param[in] gem_add_rot_energy [optional] if Gemini++ used, was rotational (Yrast) energy added to E* before decay?
 
   KVNameValueList params;
   params.SetValue("kinematics", kinematics);
   params.SetValue("geometry", geometry);
   params.SetValue("filter_type", filter_type);
   if (random_phi) params.SetValue("random_phi", "yes");
   else params.SetValue("random_phi", "no");
   if (run_number) params.SetValue("run_number", run_number.value());
   if (data_quality_audit) params.SetValue("data_quality_audit", data_quality_audit.value());
   if (gem_decay_per_event) {
      params.SetValue("gemini", gem_decay_per_event.value());
      params.SetValue("gem_add_erot", (gem_add_rot_energy ? (gem_add_rot_energy.value() ? "yes" : "no") : "no"));
   }
   // check not already in table
   auto id = find_index_in_table("filters", params);
   if (id)
      return id.value();
 
   enter_data_row("filters", params);
   return find_index_in_table("filters", params).value();
}
 
 
 
 
void KVSimDataManager::initialize_repository(repository_id id, const TString& root_dir, const TString& access, std::optional<TString> xrdsrv, std::optional<TString> xrdroot)
{
   // Initialize a new data repository with the given informations
   fRepos[id].reset(new KVDataRepository("", root_dir, access));
   if (xrdsrv) {
      fRepos[id]->ReadWithXrootd(xrdsrv.value(), xrdroot.value());
   }
}
 
 
 
 
std::pair<TString, TString> KVSimDataManager::import_file_to_repository(const TString& filepath, repository_id repo, dataset_id dset, reaction_id reac, std::optional<filter_id> filt_id)
{
   // Copy the file into the data repository and return a pair of TString {path,filename} with the path and filename in the repository.
   //
   // The path will be of the form
   //~~~
   // [model]/[reaction]/[filename]
   //~~~
   // or
   //~~~
   // [model]/[reaction]/[filter]/[filename]
   //~~~
   // for filtered files
 
   // get model id from dataset
   auto res = get_data_for_unique_index("datasets", dset);
   auto model_id = res.GetIntValue("model_id");
 
   TString filt;
   if (filt_id)
      filt.Form("filter_%d", filt_id.value());
   get_data_repository(repo).CopyFileToRepository(gSystem->BaseName(filepath), filepath, GetModelName(model_id), GetReactionName(reac), filt);
   return std::make_pair<TString, TString>(get_data_repository(repo).GetRepositoryPath(GetModelName(model_id), GetReactionName(reac), filt), gSystem->BaseName(filepath));
}
 
 
 
 
KVSimDataManager::repository_id KVSimDataManager::AddDataRepository(const TString& root_dir, const TString& access,
      std::optional<TString> xrdsrv, std::optional<TString> xrdroot)
{
   // Initialize a (possibly new) data repository and return its id
   //
   // If the same repository is not already in the database, it is added
   //
   // If the repository is not in the list of currently active repositories, it is initialized and added to the list
 
   KVNameValueList params;
   params.SetValue("type", access);
   TString _root_dir = root_dir;
   if (access == "local") {
      // make sure root directory is an absolute path
      auto rv = AbsoluteUnixPath(root_dir);
      if (!rv) {
         Error("AddDataRepository", "invalid root directory for local data repository: %s", root_dir.Data());
         return 0;
      }
      _root_dir = rv.value();
   }
   params.SetValue("root_dir", _root_dir);
   if (xrdsrv) params.SetValue("Xrdsrv", xrdsrv.value());
   if (xrdroot) params.SetValue("Xrdroot", xrdroot.value());
 
   // check repository not already in repository
   auto id = find_index_in_table("repositories", params);
 
   if (!id) {
      enter_data_row("repositories", params);
      id = find_index_in_table("repositories", params);
   }
 
   // has repository been initialised ?
   if (!fRepos[id.value()]) {
      initialize_repository(id.value(), root_dir, access, xrdsrv, xrdroot);
   }
 
   return id.value();
}
 
 
 
 
void KVSimDataManager::AddUnfilteredSimulation(const TString& filepath, Int_t events, primary_or_secondary P,
      dataset_id dataset, reaction_id reaction)
{
   // Add a file containing data from an unfiltered simulation
   //
   // \param[in] filepath full path to file
   // \param[in] events number of events in file
   // \param[in] P whether events are primary or secondary (before or after secondary decay)
   // \param[in] dataset index of dataset file is part of
   // \param[in] reaction index of reaction file corresponds to
   //
   // if ImportFilesToRepository() has been called with a valid data repository id, the file will be copied
   // to the repository, and any future access to the file will be via the data repository
 
   KVNameValueList params;
   params.SetValue("events", events);
   if (P == primary_or_secondary::primary)
      params.SetValue("type", "primary");
   else
      params.SetValue("type", "secondary");
   params.SetValue("dataset_id", dataset);
   params.SetValue("reaction_id", reaction);
 
   add_file_to_collection("AddUnfilteredSimulation", "unfiltered_files", params, filepath, dataset, reaction);
}
 
 
 
 
void KVSimDataManager::add_file_to_collection(const TString& method_name, const TString& table_name, KVNameValueList& params,
      const TString& filepath, dataset_id dataset, reaction_id reaction, std::optional<filter_id> filt_id)
{
   if (import_data_into_repository) {
      // check file not already in database first
      params.SetValue("repository_id", import_repo);
      params.SetValue("original_name", gSystem->BaseName(filepath));
      params.SetValue("original_path", gSystem->DirName(filepath));
      auto id = find_index_in_table(table_name, params);
      if (id) {
         Error(method_name, "The file %s has already been added", filepath.Data());
         return;
      }
      auto repo_path = import_file_to_repository(filepath, import_repo, dataset, reaction, filt_id);
      params.SetValue("name", repo_path.second);
      params.SetValue("path", repo_path.first);
   }
   else {
      params.SetValue("name", gSystem->BaseName(filepath));
      params.SetValue("path", gSystem->DirName(filepath));
      // check file is not already in database
      auto id = find_index_in_table(table_name, params);
      if (id) {
         Error(method_name, "The file %s has already been added", filepath.Data());
         return;
      }
   }
 
   enter_data_row(table_name, params);
}
 
 
 
void KVSimDataManager::AddFilteredSimulation(const KVSimFile& f, dataset_id dataset, reaction_id reaction)
{
   // Add a filtered simulation to the collection
 
   // get kinematics used for transform lab<->CM.
   // if the system name contains '@' it is an ad hoc kinematics
   // if not, we have to look for it in the experimental dataset database
   TString sys = f.GetSystem();
   reaction_id kinematics = 0;
   if (sys.Contains("@")) {
      KV2Body k2b(sys);
      auto kk = GetReactionId(k2b);
      if (!kk) {
         kinematics = AddReaction(k2b);
      }
      else
         kinematics = kk.value();
   }
   else {
      if (!gDataSetManager) {
         auto dsm = new KVDataSetManager;
         dsm->Init();
      }
      auto _sys = gDataSetManager->GetDataSet(f.GetDataSet())->GetDataBase("minimal")->GetSystem(sys);
      auto kk = GetReactionId(*_sys->GetKinematics());
      if (!kk) {
         kinematics = AddReaction(*_sys->GetKinematics());
      }
      else
         kinematics = kk.value();
   }
   auto filt_id = AddFilter(kinematics, f.GetDataSet(), f.GetFilterType(), f.IsRandomPhi(),
                            f.HasRun() ? f.GetRun() : std::optional<int> {},
                            f.HasDataQualityAudit() ? f.GetDataQualityAudit() : std::optional<TString> {},
                            f.IsGemini() ? f.GetGemDecayPerEvent() : std::optional<int> {});
 
   KVNameValueList params;
   params.SetValue("events", (int)f.GetEvents()); // warning: GetEvents() returns Long64_t!
   params.SetValue("dataset_id", dataset);
   params.SetValue("reaction_id", reaction);
   params.SetValue("filter_id", filt_id);
   // find simulation file which was filtered
   auto file_v = find_index_in_table("unfiltered_files", {{"name", f.GetOriginalFile()}});
   if (!file_v) // has file been moved to repository since filtering took place ?
      file_v = find_index_in_table("unfiltered_files", {{"original_name", f.GetOriginalFile()}});
   params.SetValue("unfiltered_file_id", file_v.value());
 
   add_file_to_collection("AddFilteredSimulation", "filtered_files", params, f.GetFullPathToFile(), dataset, reaction, filt_id);
}
 
 
 
 
void KVSimDataManager::AddFilesFromDirectory(dataset_id dataset, reaction_id reaction, const TString& dirpath)
{
   // Use KVSimDir to analyse files in directory, all supposed to correspond to the same reaction, and add them to the given dataset
 
   KVSimDir sim_dir("toto", dirpath);
   sim_dir.AnalyseDirectory();
 
   // unfiltered files
   for (auto _f : *sim_dir.GetSimDataList()) {
      auto f = dynamic_cast<KVSimFile*>(_f);
 
      AddUnfilteredSimulation(f->GetFullPathToFile(), f->GetEvents(),
                              f->IsPrimary() ? primary_or_secondary::primary : primary_or_secondary::secondary,
                              dataset, reaction);
   }
   // filtered files
   for (auto _f : *sim_dir.GetFiltDataList()) {
      auto f = dynamic_cast<KVSimFile*>(_f);
      AddFilteredSimulation(*f, dataset, reaction);
   }
}
 
 
 
 
void KVSimDataManager::ImportFilesToRepository(repository_id repo)
{
   // Call with the id of a previously initialised data repository (see AddDataRepository())
   // for all following files added with AddFilesFromDirectory() to be imported (copied)
   // into this data repository.
   //
   // Call with no argument (default repo_id = 0 => no repository) to stop importing.
 
   if (!repo) {
      import_data_into_repository = false;
      import_repo = 0;
      return;
   }
   // check repository id is valid
   try {
      get_data_repository(repo);
      import_data_into_repository = true;
      import_repo = repo;
   }
   catch (const std::exception& e) {
      Info("ImportFilesToRepository", "%s", e.what());
      import_data_into_repository = false;
      import_repo = 0;
   }
}
 
 
 
 
TString KVSimDataManager::GetReactionName(reaction_id id) const
{
   // for given reaction index, return a string with a name like:
   //~~~
   // "reaction64Ni58Ni32AMeV"
   // "decay64Ni170MeV"
   //~~~
 
   auto res = get_data_for_unique_index("reactions", id);
   if (res.HasParameter("nuc2")) {
      KVNucleus proj(res.GetStringValue("nuc1"));
      proj.SetE(res.GetDoubleValue("energy"));
      TString reac;
      reac.Form("reaction%s%s%.0fAMeV", res.GetStringValue("nuc1"), res.GetStringValue("nuc2"), proj.GetAMeV());
      if (res.HasParameter("impact_parameter"))
         reac.Append(Form("_b%.1f%s", res.GetDoubleValue("impact_parameter"), res.GetStringValue("ip_units")));
      return reac;
   }
   TString CN;
   CN.Form("decay%s%.0fMeV", res.GetStringValue("nuc1"), res.GetDoubleValue("energy"));
   if (res.HasParameter("spin_hbar"))
      CN.Append(Form("_%dhbar", res.GetIntValue("spin_hbar")));
   return CN;
}
 
 
 
 
TString KVSimDataManager::GetModelName(model_id id) const
{
   // for given model, return name as "AMD_1", "HIPSE_11", etc., where "_N" is the index which can be different
   // for different parameter sets of the model
 
   auto res = get_data_for_unique_index("models", id);
   return Form("%s_%d", res.GetStringValue("name"), id);
}
 
 
 
 
KVNameValueList KVSimDataManager::get_params_for_reaction(std::optional<double> opt, bool reaction, const KV2Body& R, const TString& ip_units) const
{
   KVNameValueList params;
   params.SetValue("nuc1", R.GetNucleus(1)->GetSymbol());
   if (reaction) {
      params.SetValue("nuc2", R.GetNucleus(2)->GetSymbol());
      params.SetValue("energy", R.GetNucleus(1)->GetEnergy());
      if (opt) {
         params.SetValue("impact_parameter", opt.value());
         params.SetValue("ip_units", ip_units);
      }
   }
   else {
      params.SetValue("energy", -(R.GetExcitEnergy()));
      if (opt)
         params.SetValue("spin_hbar", TMath::Nint(opt.value()));
   }
 
   return params;
}
 
 
 
 
const KVDataRepository& KVSimDataManager::get_data_repository(repository_id id)
{
   // Return reference to repository corresponding to an entry in the repositories table
   //
   // If not already initialized, the repository is initialized using the infos in the table
   //
   // If the repository can not be initialized because it is not in the database,
   // throw a std::runtime_error()
 
   if (!fRepos[id]) {
      auto make_repository = [&](const KVNameValueList & columns) {
         std::optional<TString> xrdsrv;
         if (columns.HasStringParameter("Xrdsrv"))
            xrdsrv = columns["Xrdsrv"].GetTString();
         std::optional<TString> xrdroot;
         if (columns.HasStringParameter("Xrdroot"))
            xrdroot = columns["Xrdroot"].GetTString();
         initialize_repository(id, columns["root_dir"].GetString(), columns["type"].GetString(), xrdsrv, xrdroot);
         return false;
      };
      fDB.get_name_value_list_for_each_row("repositories", make_repository, Form("id = %d", id));
   }
   if (!fRepos[id])
      throw std::runtime_error(Form("no repository with id=%d in database; you should call AddDataRepository() first", id));
   return *fRepos[id];
}
 
 
 
 
KVSimDataManager::reaction_id KVSimDataManager::AddReaction(const KV2Body& R, std::optional<double> opt, const TString& ip_units)
{
   // Add new reaction to database
   //
   // \param[in] R two-body reaction or compound nucleus definition (KV2Body)
   // \param[in] opt if given:
   //                   + for reactions, fixed value of impact parameter
   //                   + for CN decay, fixed spin of compound nucleus (integer value)
   // \param[in] ip_units for impact parameter given in opt, either "fm" (default) or "rel" (=> b/bmax)
   //
   // \returns index of reaction in database
 
   auto reaction = (R.GetNucleus(2) != nullptr);
 
   KVNameValueList params = get_params_for_reaction(opt, reaction, R, ip_units);
 
   // check not already in database
   auto id = find_index_in_table("reactions", params);
   if (id)
      return id.value();
 
   enter_data_row("reactions", params);
 
   return find_index_in_table("reactions", params).value();
}
 
 
 
 
std::optional<KVSimDataManager::reaction_id> KVSimDataManager::GetReactionId(const KV2Body& R, std::optional<double> opt, const TString& ip_units) const
{
   // \param[in] R two-body reaction or compound nucleus definition (KV2Body)
   // \param[in] opt if given:
   //                   + for reactions, fixed value of impact parameter
   //                   + for CN decay, fixed spin of compound nucleus (integer value)
   // \param[in] ip_units for impact parameter given in opt, either "fm" (default) or "rel" (=> b/bmax)
   //
   // \returns index of reaction in database, if found
 
   auto reaction = (R.GetNucleus(2) != nullptr);
   KVNameValueList params = get_params_for_reaction(opt, reaction, R, ip_units);
   return find_index_in_table("reactions", params);
}
 
 
 
 
KVSimDataManager::model_id KVSimDataManager::AddModel(const TString& name, const KVNameValueList& params)
{
   // Add model with given name and parameter values to database.
   //
   // \param[in] name (short) name of model
   // \param[in] params arbitrary list of key-value pairs giving full description of model parameters
   //
   // \return index in models table
   //
   // \note that for the same model, different parameter values have different indices.
 
   fDB.add_missing_columns("models", params); // add any missing parameter columns to table
   KVNameValueList params_copy(params);
   params_copy.SetValue("name", name);
 
   enter_data_row("models", params_copy);
 
   return GetModelId(name, params).value();
}
 
 
 
 
std::optional<KVSimDataManager::model_id> KVSimDataManager::GetModelId(const TString& name, const KVNameValueList& params) const
{
   // Look for model with given name and parameter values in database
   //
   // \param[in] name (short) name of model
   // \param[in] params arbitrary list of key-value pairs giving full description of model parameters
   //
   // \return index in models table if found, std::nullopt if not
   //
   // \note that for the same model, different parameter values have different indices.
 
   fDB.add_missing_columns("models", params); // add any missing parameter columns to table
   KVNameValueList params_copy(params);
   params_copy.SetValue("name", name);
 
   return find_index_in_table("models", params_copy);
}
 
 
 
 
KVSimDataManager::dataset_id KVSimDataManager::AddDataSet(const TString& short_description, model_id modid)
{
   // Add a new dataset for given model with a short description
   //
   // \param[in] modid index of model with a given set of parameters in the database
   // \param[in] short_description short description (1 phrase) of dataset
   //
   // \return index of dataset in table
 
   KVNameValueList params;
   params.SetValue("model_id", modid);
   params.SetValue("user", gSystem->GetUserInfo()->fUser);
   params.SetValue("description", short_description);
 
   enter_data_row("datasets", params);
 
   return find_index_in_table("datasets", params).value();
}
 
 
 
 
KVSimDataManager::dataset_id KVSimDataManager::AddDataSet(const TString& short_description, const TString& model, const KVNameValueList& model_params)
{
   // Add a new dataset for given model with a short description
   //
   // \param[in] short_description short description (1 phrase) of dataset
   // \param[in] model (short) name of model
   // \param[in] model_params arbitrary list of key-value pairs giving full description of model parameters used in dataset
   //
   // \return index of dataset in table
 
   auto modid = GetModelId(model, model_params);
   if (!modid) {
      modid = AddModel(model, model_params);
   }
   return AddDataSet(short_description, modid.value());
}