KVDataSet.cpp

/*
$Id: KVDataSet.cpp,v 1.41 2009/03/11 14:22:41 franklan Exp $
$Revision: 1.41 $
$Date: 2009/03/11 14:22:41 $
$Author: franklan $
*/
 
#include "TMethodCall.h"
#include "KVDataSet.h"
#include "KVDataSetRepository.h"
#include "KVDataRepositoryManager.h"
#include "KVDataSetManager.h"
#include "TSystem.h"
#include "TObjArray.h"
#include "TObjString.h"
#include "KVDBSystem.h"
#include "KVDBRun.h"
#include "TEnv.h"
#include "KVAvailableRunsFile.h"
#include "KVNumberList.h"
#include "TPluginManager.h"
#include "TClass.h"
#include "KVRunFile.h"
 
using namespace std;
 
ClassImp(KVDataSet)
 
KVDataSet* gDataSet;
 
 
 
KVDataSet::KVDataSet()
{
   //Default constructor
   fAllTasks.SetOwner(kTRUE);
   fTasks.SetOwner(kFALSE);
}
 
 
 
 
KVAvailableRunsFile* KVDataSet::GetAvailableRunsFile(const TString& type) const
{
   // \param type name of data type ('raw', 'recon', ...)
   // \returns pointer to available runs file object for given data type
   // \note if no data repository is associated with dataset, returns nullptr
   if (!fRepository) return nullptr;
   KVAvailableRunsFile* avrf =
      (KVAvailableRunsFile*) fAvailableRuns.FindObjectByName(type);
   if (!avrf) {
      avrf = fRepository->NewAvailableRunsFile(type, this);
      fAvailableRuns.Add(avrf);
   }
   return avrf;
}
 
 
 
 
TString KVDataSet::GetDBFileName() const
{
   //Returns name of file containing database for dataset.
   //
   //This is fixed as `DataBase.root.M.m.p` where
   //   - `M` is the current major version number
   //   - `m` is the current minor version number
   //   - `p` is the current patch number
   //
   //(fixed to be consistent with CMake function INSTALL_KALIVEDA_DATASETS which sets up
   //the Makefile for each dataset which automatically triggers rebuilding of the ROOT file
   //when source files change: thus we must have the right file name!)
   //
   //This in order to avoid problems if several different versions of KaliVeda
   //use the same working directory (in user's `$HOME/.kaliveda`) to write the
   //database files, which are often incompatible between versions
 
   TString n = "DataBase.root";
   n.Append(Form(".%d.%d.%d", GetKVMajorVersion(),
                 GetKVMinorVersion(), GetKVPatchVersion()));
   return n;
}
 
 
 
 
TString KVDataSet::GetDBName() const
{
   //\returns name of database object in database file.
   //
   //If this is not set explicitly with SetDBName(), we use the name of the dataset by default
 
   return (fDBName != "" ? fDBName.Data() : GetName());
}
 
 
 
 
 
void KVDataSet::OpenDBFile(const Char_t* full_path_to_dbfile) const
{
   //Open the database from a file on disk.
 
   TDirectory* work_dir = gDirectory;   //keep pointer to current directory
   fDBase.reset(new TFile(full_path_to_dbfile, "READ"));
 
   if (fDBase->IsOpen()) {
      fDataBase = dynamic_cast<KVExpDB*>(fDBase->Get(GetDBName()));
      if (!fDataBase) {
         KVError::Error(this, "OpenDBFile", "%s not found in file %s", GetDBName().Data(),
               GetDBFileName().Data());
      }
      else {
         fDataBase->ReadObjects(fDBase.get());   // read any associated objects
      }
      work_dir->cd();           //back to initial working directory
   }
}
 
 
 
 
 
TString KVDataSet::GetFullPathToDB() const
{
   // \returns full path to file where database is written on disk
 
   TString dbfile = GetDBFileName();
   TString dbfile_fullpath;
   TString tmp;
 
   // If this dataset is just an alias for another dataset i.e. if DataSet.Directory
   // is set with just the name of another dataset (not a full path to dataset files)
   // then the database file should be written/found under the name of the alias.
   TString dataset_alias = GetDataSetEnv("DataSet.Directory", GetName());
   TString db_alias = GetName();
   if (!gSystem->IsAbsoluteFileName(dataset_alias)) db_alias = dataset_alias;
 
   AssignAndDelete(tmp, gSystem->ConcatFileName(GetDATABASEFilePath(), db_alias.Data()));
   AssignAndDelete(dbfile_fullpath, gSystem->ConcatFileName(tmp.Data(), dbfile.Data()));
   return dbfile_fullpath;
}
 
 
 
 
void KVDataSet::SetDataSetSpecificTaskParameters(KVDataAnalysisTask* t) const
{
   //Check configuration variables to see if the task parameters have been "tweaked" for the dataset.
 
   auto envar = GetDataSetEnv<KVString>(Form("%s.DataAnalysisTask.Title", t->GetName()));
   if (envar != "") t->SetTitle(envar);
   envar = GetDataSetEnv<KVString>(Form("%s.DataAnalysisTask.Analyser", t->GetName()));
   if (envar != "") t->SetDataAnalyser(envar);
   envar = GetDataSetEnv<KVString>(Form("%s.DataAnalysisTask.UserClass.Base", t->GetName()));
   if (envar != "") t->SetUserBaseClass(envar);
   envar = GetDataSetEnv<KVString>(Form("%s.DataAnalysisTask.Prereq", t->GetName()));
   if (envar != "") t->SetPrereq(envar);
   envar = GetDataSetEnv<KVString>(Form("%s.DataAnalysisTask.UserClass.ExtraACliCIncludes", t->GetName()));
   if (envar != "") t->SetExtraAClicIncludes(envar);
   auto nev = GetDataSetEnv<int>(Form("%s.DataAnalysisTask.StatusUpdateInterval", t->GetName()));
   if (nev > 0) t->SetStatusUpdateInterval(nev);
}
 
 
 
 
 
void KVDataSet::SaveDataBase() const
{
   // Write the experiment database for this dataset to disk (ROOT file), creating and setting
   // permissions for any required directories
 
   TString dbfile_fullpath = GetFullPathToDB();
   TString tmp = gSystem->DirName(dbfile_fullpath.Data());  //full path to directory $KVROOT/db/[dataset name]
 
   if (gSystem->AccessPathName(tmp.Data())) {   // directory $KVROOT/db/[dataset name] does not exist
 
      if (gSystem->mkdir(tmp.Data()) == -1) {  // problem creating $KVROOT/db/[dataset name]
 
         TString tmp2 = gSystem->DirName(tmp.Data());// full path to directory $KVROOT/db
 
         if (gSystem->AccessPathName(tmp2.Data())) {   // directory $KVROOT/db does not exist
 
            if (gSystem->mkdir(tmp2.Data()) == -1) { // problem creating $KVROOT/db
               KVError::Error(this, "SaveDataBase", "Cannot create directory %s required to save database",
                     tmp2.Data());
               return;
            }
            gSystem->Chmod(tmp2.Data(), 0775);
         }
         else {
            KVError::Error(this, "SaveDataBase", "Cannot create directory %s required to save database, even though %s exists: check disk space ?",
                  tmp.Data(), tmp2.Data());
            return;
         }
         //try again
         if (gSystem->mkdir(tmp.Data()) == -1) {
            KVError::Error(this, "SaveDataBase", "Cannot create directory %s required to save database",
                  tmp.Data());
            return;
         }
         else {
            gSystem->Chmod(tmp.Data(), 0775);
         }
      }
      else {
         gSystem->Chmod(tmp.Data(), 0775);
      }
   }
 
   WriteDBFile(dbfile_fullpath.Data());
}
 
 
 
 
 
void KVDataSet::WriteDBFile(const Char_t* full_path_to_dbfile) const
{
   //Write the database to disk.
 
   TDirectory* work_dir = gDirectory;   //keep pointer to current directory
   if (!fDataBase) {
      KVError::Error(this, "WriteDBFile", "Database has not been built");
      return;
   }
   fDBase.reset(new TFile(full_path_to_dbfile, "recreate"));
   fDBase->cd();                //set as current directory (maybe not necessary)
   fDataBase->Write(GetDBName());    //write database to file with given name
   fDataBase->WriteObjects(fDBase.get());   //write any associated objects
   fDBase->Write();        // write file header etc.
   fDBase->Close();         // close file
   gSystem->Chmod(full_path_to_dbfile, 0664); // set permissions to rw-rw-r--
   work_dir->cd();              //back to initial working directory
}
 
 
 
 
KVExpDB* KVDataSet::GetDataBase(const TString& opt) const
{
   //\returns pointer to database associated with this dataset.
   //
   //Opens, updates or creates database file if necessary
   //(the database is automatically rebuilt if the source files are
   //more recent than the last database file).
   //
   //\param[in] opt optional option string:
   //                 - if opt="update": force regeneration of the database from source files in dataset directory
   //                 - if opt="minimal": if database not built/out of date, only build runs/systems database
   //                 - note that both options can be given to force update of minimal database
 
   OpenDataBase(opt);
   return fDataBase;
}
 
 
 
 
void KVDataSet::OpenDataBase(const TString& opt) const
{
   //Open the database for this dataset.
   //
   //If the database does not exist or is older than the source files
   //the database is automatically rebuilt (see DataBaseNeedUpdate()).
   //
   //\param[in] opt build option
   //
   //  - if opt="update" we force rebuilding of the database
   //  - if opt="minimal" only a minimal database of runs & systems for the dataset is built
   //  - note that both options can be given to force update of minimal database
 
   TString _opt(opt);
   _opt.ToUpper();
   auto update = (_opt.Contains("UPDATE"));
   auto minimal = (_opt.Contains("MINIMAL"));
 
   auto rebuild_database =  [&](bool start_from_minimal = false) {
      start_from_minimal = false;//never start from minimal
      if(start_from_minimal)
         Info("OpenDataBase", "Updating existing minimal database...");
      else
      {
         if(minimal)
            Info("OpenDataBase", "Creating a new minimal database...");
         else
            Info("OpenDataBase", "Creating a new full database...");
      }
 
      fDataBaseUpdateInProgress = true;
      //check if it is the currently active database (gDataBase),
      //in which case we must 'cd()' to it after rebuilding
      auto is_glob_db = (fDataBase == gExpDB);
      if(!start_from_minimal)
      {
         if (fDataBase) {
            delete fDataBase;
            fDataBase = 0;
         }
         fDataBase = KVExpDB::MakeDataBase(GetDBName(), GetDataSetDir(), minimal);
         if (!fDataBase) {
            // no database defined for dataset
            Info("OpenDataBase", "No database defined for dataset");
            fDataBaseUpdateInProgress = false;
            return;
         }
      }
      else
         fDataBase->CompleteDataBase();
      SaveDataBase();
      if (fDataBase && is_glob_db) fDataBase->cd();
      fDataBaseUpdateInProgress = false;
   };
 
   if (update || DataBaseNeedsUpdate()) {
      //if option="update" or database out of date or does not exist, (re)build the database
      rebuild_database();
   }
   else if (!fDataBase) {
      // if database is not in memory at this point, we need to
      // open the database file and read in the database
 
      //load plugin for database
      if (!LoadPlugin("KVExpDB", GetDBName())) {
         KVError::Error(this, "GetDataBase", "Cannot load required plugin library");
         return;
      }
      //look for database file in dataset subdirectory
      TString dbfile_fullpath = GetFullPathToDB();
      //open database file
      OpenDBFile(dbfile_fullpath.Data());
 
      // if the previously built database was minimal but nobody asked for
      // a minimal database (=> they want the full monty), we rebuild a full database
      if (fDataBase && fDataBase->IsMinimal() && !minimal)
         rebuild_database(true);
   }
   else if (fDataBase && fDataBase->IsMinimal() && !minimal) {
      // database is already open, but was only a minimal build
      // now we want the full monty!
      rebuild_database(true);
   }
}
 
 
 
 
 
void KVDataSet::ls(Option_t*) const
{
   //Print dataset information
   cout << "Dataset name=" << GetName() << " (" << GetTitle() << ")";
   if (IsAvailable()) {
      cout << "  [ AVAILABLE: ";
      cout << fDatatypes.Data();
      cout << "]";
   }
   else
      cout << "  [UNAVAILABLE]";
   cout << endl;
}
 
 
 
 
void KVDataSet::Print(Option_t* opt) const
{
   //Print dataset information
   //
   //param[in] opt select optional output formats:
   //            - if string contains "tasks", print numbered list of tasks that can be performed
   //            - if string contains "data", print list of available data types
 
   TString Sopt(opt);
   Sopt.ToUpper();
   if (Sopt.Contains("TASK")) {
      if (!GetNtasks()) {
         cout << "                    *** No available analysis tasks ***"
              << endl;
         return;
      }
      else {
         for (int i = 1; i <= GetNtasks(); i++) {
            KVDataAnalysisTask* dat = GetAnalysisTask(i);
            cout << "\t" << i << ". " << dat->GetTitle() << endl;
         }
      }
      cout << endl;
   }
   else if (Sopt.Contains("DATA")) {
      cout << "Available data types: " << fDatatypes.Data() << endl;
   }
   else {
      ls(opt);
   }
}
 
 
 
 
void KVDataSet::CheckAvailable()
{
   //Check if this data set is physically present and available for analysis.
   //
   //In other words we check if the value of GetDataPathSubdir() is a subdirectory
   //of the current data repository.
   //If so, we proceed to check for the existence of sudirectories corresponding
   // to any of the datatypes associated with the dataset.
   //
 
   if (!fRepository)   // for a stand-alone KVDataSetManager not linked to a KVDataRepository,
      SetAvailable();   // all known datasets are 'available'
   else
      SetAvailable(fRepository->CheckSubdirExists(GetDataPathSubdir()));
   if (!IsAvailable())
      return;
   //check subdirectories
   auto data_types = GetDataSetEnv<KVString>("KVDataSet.DataTypes");
   if (data_types == "") {
      KVError::Warning(this, "CheckAvailable", "No datatypes defined for this dataset: %s\nCheck value of KVDataSet.DataTypes or %s.KVDataSet.DataTypes",
              GetName(), GetName());
      SetAvailable(kFALSE);
   }
   fDatatypes = "";
   // loop over data types
   data_types.Begin(" ");
   while (!data_types.End()) {
      KVString type = data_types.Next(kTRUE);
      if (!fRepository ||
            (fRepository && fRepository->CheckSubdirExists(GetDataPathSubdir(), GetDataTypeSubdir(type.Data())))
         ) {
         AddAvailableDataType(type.Data());
      }
   }
   //check at least one datatype exists
   SetAvailable(fDatatypes != "");
}
 
 
 
 
void KVDataSet::AddAvailableDataType(const Char_t* type)
{
   if (fDatatypes != "") fDatatypes += " ";
   KVString _type = type;
   _type.Remove(TString::kBoth, ' '); //strip whitespace
   fDatatypes += _type;
}
 
 
 
 
void KVDataSet::SetAnalysisTasks(const KVSeqCollection* task_list)
{
   // Add to fAllTasks list any data analysis task in list 'task_list'
   //
   // Add to fTasks list any data analysis task in list 'task_list' whose pre-requisite datatype is present for this dataset.
   //
   // Any dataset-specific "tweaking" of the task (including the prerequisite datatype) is done here.
 
   TString availables = gEnv->GetValue(Form("%s.DataAnalysisTask", GetName()), "");
   fAllTasks.Clear();
   fTasks.Clear();
   TIter nxt(task_list);
   KVDataAnalysisTask* dat;
   while ((dat = (KVDataAnalysisTask*) nxt())) {
      //make new copy of default analysis task
      if (availables == "" || availables.Contains(dat->GetName())) {
         KVDataAnalysisTask* new_task = new KVDataAnalysisTask(*dat);
         //check if any dataset-specific parameters need to be changed
         SetDataSetSpecificTaskParameters(new_task);
         fAllTasks.Add(new_task);
         // add tasks with available prerequisite data to fTasks
         if (HasDataType(new_task->GetPrereq())) {
            fTasks.Add(new_task);
         }
      }
   }
}
 
 
 
 
Int_t KVDataSet::GetNtasks() const
{
   //Returns the number of tasks associated to dataset which are compatible
   //with the available data
 
   return fTasks.GetSize();
}
 
 
 
 
KVDataAnalysisTask* KVDataSet::GetAnalysisTask(Int_t k) const
{
   //Return kth analysis task in list of available tasks.
   //
   //\param[in] k task number in range [1, GetNtasks()]. Corresponds to the number shown next to the title of the task when Print("tasks") is called
   return (KVDataAnalysisTask*) fTasks.At(k - 1);
}
 
 
 
 
KVUnownedList KVDataSet::GetListOfAvailableSystems(const TString& datatype)
{
   //\returns list of available systems for this dataset and the given datatype
   //
   //\param[in] datatype type of data
 
   if (!GetAvailableRunsFile(datatype)) {
      KVError::Error(this, "GetListOfAvailableSystems(const Char_t*)",
            "No available runs file for type %s", datatype.Data());
      return {};
   }
   return GetAvailableRunsFile(datatype)->GetListOfAvailableSystems();
}
 
 
 
 
KVUnownedList KVDataSet::GetListOfAvailableSystems(KVDataAnalysisTask* datan)
{
   //\returns list of available systems for this dataset and the prerequisite datatype for the given analysis task
   //
   //\param[in] datan data analysis task
 
   return GetListOfAvailableSystems(datan->GetPrereq());
}
 
 
 
 
KVList KVDataSet::GetListOfAvailableRunFilesForSystem(const TString& datatype, KVDBSystem* systol)
{
   //\returns list of available runfiles for this dataset and the given datatype and system.
   //
   //\param[in] datatype type of data
   //\param[in] systol pointer to system
   //
   //If no systems are defined for the dataset then we return a list of available runfiles for the given datatype
 
   if (!GetAvailableRunsFile(datatype)) {
      KVError::Error(this, "GetListOfAvailableSystems(const Char_t*)",
            "No available runs file for type %s", datatype.Data());
      return {};
   }
   return GetAvailableRunsFile(datatype)->GetListOfAvailableRunFilesForSystem(systol);
}
 
 
 
 
KVList KVDataSet::GetListOfAvailableRunFilesForSystem(KVDataAnalysisTask* datan, KVDBSystem* systol)
{
   //\returns list of available runfiles for this dataset, given system, and the prerequisite datatype for the given analysis task
   //
   //\param[in] datan data analysis task
   //\param[in] pointer to system
   //
   //If no systems are defined for the dataset then we return a list of available runfiles for the given datatype
 
   return GetListOfAvailableRunFilesForSystem(datan->GetPrereq(), systol);
}
 
 
 
 
KVUniqueNameList KVDataSet::GetSystemsForRunFiles(const TString& datatype, const run_index_list& rl) const
{
   // \returns the list of systems corresponding to the given list of runfiles & datatype
 
   if (!GetAvailableRunsFile(datatype)) {
      KVError::Error(this, "GetSystemsForRunFiles",
            "No available runs file for type %s", datatype.Data());
      return {};
   }
   return GetAvailableRunsFile(datatype)->GetSystemsForRunFiles(rl);
}
 
 
 
 
void KVDataSet::SetName(const char* name)
{
   // Set name of dataset
   //
   // Also sets path to directory containing database informations
   // for this dataset, i.e. list of runs, systems, calibration files etc.
   //
   //   By default, just the name of the dataset is used, i.e.
   //   `[DATADIR]/name`
   //   (where `DATADIR` = path given by KVBase::GetDATADIRFilePath())
   //
   //   However, if the variable
   //~~~
   //  [name].DataSet.Directory:   [path]
   //~~~
   //   has been set, the value of `[path]` will be used:
   //      - if [path] is an absolute path name, it will be used as such
   //      - if [path] is an incomplete or relative path, it will be prepended with `[DATADIR]/`
   //
   // This allows to use one dataset as an alias for another, by setting `DataSet.Directory`
   // to the name of an existing dataset
   TNamed::SetName(name);
   TString path = GetDataSetEnv("DataSet.Directory", name);
   if (gSystem->IsAbsoluteFileName(path)) fCalibDir = path;
   else {
      // in this case (not an absolute path but just the name of another dataset)
      // this dataset is an alias for another dataset.
      fCalibDir = GetDATADIRFilePath(path);
      // the name of the database object is the name of the "true" dataset
      SetDBName(path);
   }
}
 
 
 
 
const Char_t* KVDataSet::GetDataSetDir() const
{
   //\returns full path to directory containing database and calibration/identification parameters etc. for this dataset.
 
   return fCalibDir.Data();
}
 
 
 
 
void KVDataSet::cd() const
{
   // Makes this dataset the "currently active" or default dataset.
   //
   //At the same time, the database and data repository associated with
   //this dataset also become the "currently active" ones:
   //
   //  | global pointer | represents | base class |
   //  |----------------|------------|------------|
   //  | `gDataSet` | active dataset | KVDataSet |
   //  | `gExpDB` | associated experimental database | KVExpDB |
   //  | `gDataRepository` | repository containing runfiles | KVDataRepository |
   //
 
   gDataSet = const_cast<KVDataSet*>(this);
   if (fRepository) fRepository->cd();
   KVExpDB* db = GetDataBase();
   if (db) db->cd();
}
 
 
 
 
TObject* KVDataSet::open_runfile(const KVString& type, const run_index_t& run)
{
   // Open file containing data of given datatype for given run number of this dataset.
   //
   // \returns a pointer to the opened file; if the file is not available, we return nullptr.
   //
   // The user must cast the returned pointer to the correct class, which will
   // depend on the data type and the dataset
   //
   // **SPECIAL CASE: MFM data with EBYEDAT frames**
   //
   // If the variable
   //
   //~~~~~~~~~~~~~~~~~~~~~~~~~
   //   [dataset].MFM.WithEbyedat:   yes
   //~~~~~~~~~~~~~~~~~~~~~~~~~
   //
   // is set, then we expect to find the necessary `ACTIONS_*` files in the dataset directory
   // in subdirectory `ebyedat` (they should have the same names as the data files prefixed by
   // `ACTIONS_[expname].CHC_PAR.`).
   //
   // If in addition the variable
   //
   //~~~~~~~~~~~~~~~~~~~~~~~~~
   //   [dataset].MFM.EbyedatActionsExpName:   [expname]
   //~~~~~~~~~~~~~~~~~~~~~~~~~
   //
   // is set, then we use the same `ACTIONS` file for all runs, with name `ACTIONS_[expname].CHC_PAR`
 
   if (!GetRepository()) return nullptr;
 
   auto file_open = [&](const TString & fpth) {
      return file_opener(type, fpth, GetName());
   };
 
   if (type=="raw" && GetDataSetEnv<KVString>("MFM.WithEbyedat")=="yes") {
      TString ebydir = GetDataSetDir();
      ebydir += "/ebyedat";
      gEnv->SetValue("KVMFMDataFileReader.ActionsDirectory", ebydir);
      if (GetDataSetEnv<KVString>("MFM.EbyedatActionsExpName")!="")
         gEnv->SetValue("KVMFMDataFileReader.ActionsExpName", GetDataSetEnv<KVString>("MFM.EbyedatActionsExpName"));
      TObject* f = GetRepository()->OpenFile(GetRunfileName(type, run), file_open, GetDataPathSubdir(), GetDataTypeSubdir(type));
      // reset in case another dataset opens a raw MFM file without EBYEDAT data
      gEnv->SetValue("KVMFMDataFileReader.ActionsDirectory", "");
      gEnv->SetValue("KVMFMDataFileReader.ActionsExpName", "");
      return f;
   }
   return GetRepository()->OpenFile(GetRunfileName(type, run), file_open, GetDataPathSubdir(), GetDataTypeSubdir(type));
}
 
 
 
 
 
TString KVDataSet::GetFullPathToRunfile(const KVString& type,
                                        const run_index_t& run) const
{
   //\return full path to file containing data of given datatype for given run/index of this dataset
   // \note only works for available run files, if their is no file in the repository for this run, the returned path will be empty
 
   TString file("");
   if (fRepository) file = GetRunfileName(type, run);
   if (file == "")
      return file.Data();
   return fRepository->GetFullPathToOpenFile(file, GetDataPathSubdir(), GetDataTypeSubdir(type));
}
 
 
 
 
 
KVString KVDataSet::GetRunfileName(const KVString& type, const run_index_t& run) const
{
   //\return name of file containing data of given datatype for given run/index of this dataset
   //\note only works for available run files, if there is no file in the repository for this run, the returned path will be empty
 
   if (!HasDataType(type)) {
      KVError::Error(this, "GetRunfileName",
            "No data of type \"%s\" available for dataset %s", (const char*)type,
            GetName());
      return 0;
   }
   //get name of file from available runs file
   return GetAvailableRunsFile(type)->GetFileName(run);
}
 
 
 
 
 
std::optional<TDatime> KVDataSet::GetRunfileDate(const KVString& type, const run_index_t& run)
{
   //\return date of file containing data of given datatype for given run/index of this dataset
   //
   //\note only works for available runfiles, if there is no file in the repository for this run/index,
   //an error will be printed and std::optional will not contain a value
 
   if (!HasDataType(type)) {
      KVError::Error(this, "GetRunfileDate",
            "No data of type \"%s\" available for dataset %s", (const char*)type,
            GetName());
      return std::nullopt;
   }
   //get date of file from available runs file
   TDatime date;
   TString filename;
   if (!GetAvailableRunsFile(type)->GetRunInfo(run, date, filename)) {
      KVError::Error(this, "GetRunfileDate",
            "Runfile not found for run %d index %d (data type: %s)", run.run(), run.index(), (const char*)type);
      return std::nullopt;
   }
   return date;
}
 
 
 
 
std::optional<run_index_t> KVDataSet::GetRunNumberFromFileName(const TString& datatype, const TString& filename)
{
   // \returns run number and eventual index by parsing the name of a runfile for given datatype. if filename does not have correct format, returns std::nullopt
   //
   // If the runfile in question was generated using an index multiplier, e.g. if it contains the number
   // '17001' when the index multiplier defined for the dataset is 1000, we return '17' as the
   // run number and '1' as the index.
   //
   // Usage examples (using global pointer gDataSet to currently 'active' dataset):
   //~~~{.cpp}
   //auto run = gDataSet->GetRunNumberFromFileName(filename);
   //
   //if(!run) // no value (std::nullopt) returned
   //{
   //   std::cerr << "Filename " << filename << " is not right format\n";
   //}
   //else
   //{
   //   std::cout << "Filename " << filename << " corresponds to run number " << run.value().run();
   //   if(run.has_index()) std::cout << ", file index = " << run.index();
   //   std::cout << std::endl;
   //}
   //~~~
 
   auto arf = GetAvailableRunsFile(datatype);
   auto v = arf->IsRunFileName(filename);
   if (!v) return v;
   // if the returned value has a non-null index, there is nothing more to do
   if (v.value().has_index()) return v;
   int r = v.value().run();
   // check if returned run number actually consists of run*multiplier+index
   auto db = GetDataBase("minimal");
   if (db->HasIndexMultiplier() && r > db->GetIndexMultiplier()) {
      r /= db->GetIndexMultiplier();
      int i = v.value().run() % db->GetIndexMultiplier();
      return run_index_t{r, i};
   }
   return v;
}
 
 
 
 
 
Bool_t KVDataSet::CheckRunfileAvailable(const KVString& type, const run_index_t& run)
{
   //We check the availability of the run by looking in the available runs file associated
   //with the given datatype.
 
   //check data type is available
   if (!HasDataType(type)) {
      KVError::Error(this, "CheckRunfileAvailable",
            "No data of type \"%s\" available for dataset %s", (const char*)type,
            GetName());
      return 0;
   }
   return GetAvailableRunsFile(type)->CheckAvailable(run);
}
 
 
 
 
 
TString KVDataSet::GetBaseFileName(const Char_t* type, const run_index_t& run) const
{
   //PRIVATE METHOD: Returns base name of data file containing data for the run of given datatype.
   //The filename corresponds to one of the formats defined in $KVROOT/KVFiles/.kvrootrc
   //by variables like:
   //
   //~~~
   //[dataset].DataSet.RunFileName.[type]:    run%R.dat
   //~~~
   //
   //%R will be replaced with the run number
   //
   //IF the format contains '%D' it will be replaced with the current date and time
   //
   // Any index will be appended at the end: ".index"
 
   TString tmp;
   //get format string
   auto fmt = GetDataSetEnv<TString>(Form("DataSet.RunFileName.%s", type));
   TString run_num(Form("%d", run.run()));
   KVDatime now;
   TString date(now.AsSQLString());
   tmp = fmt;
   tmp.ReplaceAll("%R", run_num);
   if (fmt.Contains("%D")) {
      tmp.ReplaceAll("%D", date);
   }
   if (run.has_index()) tmp += Form(".%d", run.index());
   return tmp;
}
 
 
 
 
 
void KVDataSet::UpdateAvailableRuns(const KVString& type)
{
   //Update list of available runs for given data 'type'
   //
   //As we never clear the sqlite database of any previously existing runfiles
   //beforehand, it is possible that newer versions of said files have been
   //added and that we will therefore end up with multiple copies of files in the
   //database. Therefore we perform an automatic 'CleanRunfileDataBase' afterwards,
   //but only to remove runfiles which no longer physically exist
 
   //check data type is available
   if (!HasDataType(type)) {
      KVError::Error(this, "UpdateAvailableRuns",
            "No data of type \"%s\" available for dataset %s", (const char*)type,
            GetName());
   }
   KVAvailableRunsFile* a = GetAvailableRunsFile(type);
   a->Update();
   CleanRunfileDataBase(type);
}
 
 
 
 
 
TFile* KVDataSet::NewRunfile(const KVString& type, const run_index_t& run)
{
   // Create a new runfile for the dataset of given datatype.
   // (only if this dataset is associated with a data repository)
   // Once the file has been filled, use CommitRunfile to submit it to the repository.
 
   if (!fRepository) return nullptr;
   TString tmp = GetBaseFileName(type, run);
   //turn any spaces into "_"
   tmp.ReplaceAll(" ", "_");
   return fRepository->CreateNewFile(this, type, tmp.Data());
}
 
 
 
 
 
void KVDataSet::DeleteRunfile(const KVString& type, const run_index_t& run, Bool_t confirm)
{
   // Delete the file for the given run/index of data type "type" from the repository.
   // By default, confirm=kTRUE, which means that the user will be asked to confirm
   // that the file should be deleted. If confirm=kFALSE, no confirmation will be asked
   // for and the file will be deleted straight away.
   //
   // WARNING: this really does DELETE files in the repository, they cannot be
   // retrieved once they have been deleted.
 
   if (!fRepository) return;
 
   //get name of file to delete
   TString filename = GetAvailableRunsFile(type)->GetFileName(run);
   if (filename == "") {
      KVError::Error(this, "DeleteRunfile", "Run %s of type %s does not exist.",
            run.as_string().Data(), (const char*)type);
      return;
   }
   //delete file
   //prevent accidental deletion of certain types of runfiles
   auto doNotDelete = GetDataSetEnv<KVString>("DataSet.RunFile.DoNotDelete", "all");
   if (doNotDelete == "all" || doNotDelete.Contains(type)) {
      KVError::Error(this, "DeleteRunFile", "%s files cannot be deleted", (const char*)type);
      return;
   }
   fRepository->DeleteFile(this, type, filename, confirm);
   //was file deleted ? if so, remove entry from available runs file
   if (!fRepository->CheckFileStatus(this, type, filename.Data()))
      GetAvailableRunsFile(type)->Remove(run);
}
 
 
 
 
 
void KVDataSet::DeleteRunfiles(const Char_t* type, const run_index_list& nl, Bool_t confirm)
{
   // Delete files corresponding to a list of runs/index of data type "type" from the repository.
   //
   // By default, confirm=kTRUE, which means that the user will be asked to confirm
   // that each file should be deleted.
   //
   // If confirm=kFALSE, no confirmation will be asked for and the file will be deleted straight away.
   //
   // if "nl" is empty (default value) all runs of the dataset corresponding to the given type will be deleted
   //
   // WARNING: this really does DELETE files in the repository, they cannot be retrieved once they have been deleted.
 
   auto NL = nl;
   if (NL.IsEmpty()) NL = GetRunList(type);
   if (NL.IsEmpty()) return;
   for (auto& r : NL)
      DeleteRunfile(type, r, confirm);
}
 
 
 
 
 
run_index_list KVDataSet::GetRunList_DateSelection(const TString& type, const KVUnownedList& systems, std::optional<KVDatime> min_date, std::optional<KVDatime> max_date)
{
   // Prints out and returns list of runs after date / time selection
   //
   // Runs generated between ]min;max[ are selected
   //    - if min=NULL  runs with date <max are selected
   //    - if max=NULL runs with date >min are selected
   //    - if max and min are NULL returns empty list
 
   if (!min_date && !max_date) return {};
 
   if (min_date) printf("date minimum %s\n", min_date.value().AsString());
   if (max_date) printf("date maximum %s\n", max_date.value().AsString());
 
   run_index_list numb;
 
   for (auto _sys : systems) {
      auto sys = (KVDBSystem*)_sys;
      auto lrun = GetListOfAvailableRunFilesForSystem(type, sys);
 
      for (auto _run : lrun) {
         auto run = (KVRunFile*)_run;
 
         if (min_date && max_date) {
            if (min_date.value() < run->GetFileWrittenDatime() && run->GetFileWrittenDatime() < max_date.value()) {
               numb.Add(run->GetRunIndex());
            }
         }
         else if (min_date) {
            if (min_date.value() < run->GetFileWrittenDatime()) {
               numb.Add(run->GetRunIndex());
            }
         }
         else if (max_date) {
            if (run->GetFileWrittenDatime() < max_date.value()) {
               numb.Add(run->GetRunIndex());
            }
         }
      }
   }
   return numb;
 
}
 
 
 
 
 
run_index_list KVDataSet::GetRunList_StageSelection(const TString& type, const TString& ref_type, KVDBSystem* system, Bool_t OnlyCol)
{
   // Returns list of runs which are present for data type "base_type" but not for "other_type"
   //
   // If pointer to system is given, only runs for the system are considered.
   //
   // If OnlyCol=kTRUE (kFALSE default) only systems with KVDBSystem::IsCollision()=kTRUE are considered
 
   run_index_list manquant;
   auto ll = GetListOfAvailableSystems(ref_type);
   if (ll.IsEmpty()) {
      //numb.Clear();
      Info("GetRunList_StageSelection", "No data available of type \"%s\"", ref_type.Data());
      return manquant;
   }
   if (system && !ll.FindObject(system)) {
      Info("GetRunList_StageSelection", "No data available of type \"%s\" for system %s", ref_type.Data(), system->GetName());
      return manquant;
   }
 
   Info("GetRunList_StageSelection", "Liste des runs presents dans \"%s\" mais absent dans \"%s\"", ref_type.Data(), type.Data());
 
   KVDBSystem* sys = 0;
 
   for (Int_t nl = 0; nl < ll.GetEntries(); nl += 1) {
 
      sys = (KVDBSystem*)ll.At(nl);
      if (system && sys != system) continue;
      if (OnlyCol && !sys->IsCollision()) continue;
      auto nsys = GetRunList(type, sys);
      auto nsys_ref = GetRunList(ref_type, sys);
      Int_t nref = nsys_ref.GetNValues();
 
      nsys_ref.Remove(nsys);
 
      Info("GetRunList_StageSelection", "\nKVDBSystem : %s --> %d runs manquants sur %d : %s",
           sys->GetName(),
           nsys_ref.GetNValues(),
           nref,
           nsys_ref.AsString().Data()
          );
      manquant.Add(nsys_ref);
   }
   return manquant;
}
 
 
 
 
run_index_list KVDataSet::GetRunList_StageSelection(const TString& type, const TString& ref_type, const KVUnownedList& systems, Bool_t OnlyCol)
{
   // \returns list of runfiless which are present for data type "base_type" but not for "other_type"
   // for the systems (KVDBSystem pointers) in the list.
   //
   // If OnlyCol=kTRUE (kFALSE default) only systems with KVDBSystem::IsCollision()=kTRUE are considered
 
   run_index_list manquant;
   auto ll = GetListOfAvailableSystems(ref_type);
   if (ll.IsEmpty()) {
      //numb.Clear();
      Info("GetRunList_StageSelection", "No data available of type \"%s\"", ref_type.Data());
      return manquant;
   }
 
   Info("GetRunList_StageSelection", "Liste des runs presents dans \"%s\" mais absent dans \"%s\"", ref_type.Data(), type.Data());
 
   KVDBSystem* sys = 0;
 
   for (Int_t nl = 0; nl < ll.GetEntries(); nl += 1) {
 
      sys = (KVDBSystem*)ll.At(nl);
      if (!systems.Contains(sys)) continue;
      if (OnlyCol && !sys->IsCollision()) continue;
      auto nsys = GetRunList(type, sys);
      auto nsys_ref = GetRunList(ref_type, sys);
      Int_t nref = nsys_ref.GetNValues();
 
      nsys_ref.Remove(nsys);
 
      Info("GetRunList_StageSelection", "\nKVDBSystem : %s --> %d runs manquants sur %d : %s",
           sys->GetName(),
           nsys_ref.GetNValues(),
           nref,
           nsys_ref.AsString().Data()
          );
      manquant.Add(nsys_ref);
   }
   return manquant;
}
 
 
 
 
run_index_list KVDataSet::GetRunList_VersionSelection(const TString& type, const TString& version, KVDBSystem* sys)
{
   // Returns list of runs of given type that were created with the given version of KaliVeda.
   //
   // If system!="" then only runs for the given system are considered
 
   run_index_list runs;
   if (sys) {
      auto lrun = GetListOfAvailableRunFilesForSystem(type, sys);
      TIter next(&lrun);
      KVRunFile* run;
      while ((run = (KVRunFile*)next())) {
         if (!strcmp(run->GetVersion(), version)) runs.Add(run->GetRunIndex());
      }
      return runs;
   }
   auto ll = GetListOfAvailableSystems(type);
   if (ll.IsEmpty()) {
      //numb.Clear();
      Info("GetRunList_VersionSelection", "No data available of type \"%s\"", type.Data());
      return runs;
   }
   Int_t nsys = ll.GetEntries();
   for (Int_t nl = 0; nl < nsys; nl += 1) {
      sys = (KVDBSystem*)ll.At(nl);
      auto lrun = GetListOfAvailableRunFilesForSystem(type, sys);
      TIter next(&lrun);
      KVRunFile* run;
      while ((run = (KVRunFile*)next())) {
         if (!strcmp(run->GetVersion(), version)) runs.Add(run->GetRunIndex());
      }
   }
   return runs;
}
 
 
 
 
void KVDataSet::CommitRunfile(const KVString& type, const run_index_t& run, TFile* file)
{
   // Commit a runfile previously created with NewRunfile() to the repository.
   // Any previous version of the runfile will be deleted.
   // The available runs list for this data 'type'  is updated.
 
   if (!fRepository) return;
 
   //keep name of file for updating available runs list
   TString newfile = gSystem->BaseName(file->GetName());
 
   fRepository->CommitFile(file, type, this);
   //update list of available datatypes of dataset,
   //in case this addition has created a new subdirectory
   CheckAvailable();
   //check if previous version of file exists
   //get name of file from available runs file
   //note that when the file is the first of a new subdirectory, GetAvailableRunsFile->GetFileName
   //will cause the available runs file to be created, and it will contain one entry:
   //the new file!
   TString oldfile = GetAvailableRunsFile(type)->GetFileName(run);
   if (oldfile != "" && oldfile != newfile) {
      //delete previous version - no confirmation
      fRepository->DeleteFile(this, type, oldfile.Data(),
                              kFALSE);
      //was file deleted ? if so, remove entry from available runs file
      if (!fRepository->CheckFileStatus(this, type, oldfile.Data()))
         GetAvailableRunsFile(type)->Remove(run);
   }
   if (oldfile != newfile) {
      //add entry for new run in available runs file
      GetAvailableRunsFile(type)->Add(run, newfile.Data());
   }
}
 
 
 
 
 
void KVDataSet::SetRepository(KVDataSetRepository* dr)
{
   //Set pointer to data repository in which dataset is stored
   fRepository = dr;
}
 
 
 
 
 
KVDataSetRepository* KVDataSet::GetRepository() const
{
   //Get pointer to data repository in which dataset is stored
   return fRepository;
}
 
 
 
 
 
void KVDataSet::CheckMultiRunfiles(const Char_t* data_type)
{
   //Check all runs for a given datatype and make sure that only one version
   //exists for each runfile. If not, we print a report on the runfiles which occur
   //multiple times, with the associated date and file name.
 
   auto doubles = GetAvailableRunsFile(data_type)->CheckMultiRunfiles();
   if (doubles.empty()) {
      cout << "OK. No runs appear more than once." << endl;
   }
   else {
      cout << "Runs which appear more than once: " << endl << endl;
      //print dates and filenames for each run
 
      for (auto& rr : doubles) {
         KVList filenames, dates;
 
         //get infos for current run
         GetAvailableRunsFile(data_type)->GetRunInfos(rr, dates, filenames);
 
         cout << "Run " << rr << " : " << dates.GetEntries() << " files >>>>>>" << endl;
         for (int i = 0; i < dates.GetEntries(); i++) {
 
            cout << "\t" << ((TObjString*) filenames.At(i))->String().
                 Data() << "\t" << ((TObjString*) dates.At(i))->String().
                 Data() << endl;
 
         }
      }
   }
}
 
 
 
 
 
void KVDataSet::CleanMultiRunfiles(const Char_t* data_type, Bool_t confirm)
{
   // Check all runs for a given datatype and make sure that only one version
   // exists for each run. If not, we print a report on the runfiles which occur
   // multiple times, with the associated date and file name, and then we
   // destroy all but the most recent version of the file in the repository, and
   // update the runlist accordingly.
   //
   // By default, we ask for confirmation before deleting each file.
   //
   // Call with confirm=kFALSE to delete WITHOUT CONFIRMATION (DANGER!! WARNING!!!)
 
   if (!fRepository) return;
 
   KVAvailableRunsFile* ARF = GetAvailableRunsFile(data_type);
   auto doubles = ARF->CheckMultiRunfiles();
   if (doubles.empty()) {
      cout << "OK. No runs appear more than once." << endl;
   }
   else {
      cout << "Runs which appear more than once: " << endl << endl;
      //print dates and filenames for each run
 
      KVList filenames, dates;
      for (auto& rr : doubles) {
         //get infos for current run
         ARF->GetRunInfos(rr, dates, filenames);
         TDatime most_recent("1998-12-25 00:00:00");
         Int_t i_most_recent = 0;
         cout << "Run " << rr << " : " << dates.GetEntries() << " files >>>>>>" << endl;
         for (int i = 0; i < dates.GetEntries(); i++) {
            //check if run is most recent
            TDatime rundate(((TObjString*) dates.At(i))->String().Data());
            if (rundate > most_recent) {
               most_recent = rundate;
               i_most_recent = i;
            }
         }
         //Now, we loop over the list again, this time we destroy all but the most recent
         //version of the runfile
         for (int i = 0; i < dates.GetEntries(); i++) {
            if (i == i_most_recent) {
               cout << "KEEP : ";
            }
            else {
               cout << "DELETE : ";
            }
            cout << "\t" << ((TObjString*) filenames.At(i))->String().
                 Data() << "\t" << ((TObjString*) dates.At(i))->String().
                 Data() << endl;
            if (i != i_most_recent) {
               //delete file from repository forever and ever
               fRepository->DeleteFile(this, data_type,
                                       ((TObjString*) filenames.At(i))->
                                       String().Data(), confirm);
               //remove file entry from available runlist
               ARF->Remove(rr,
                           ((TObjString*) filenames.At(i))->String());
            }
         }
      }
   }
}
 
 
 
 
void KVDataSet::CleanRunfileDataBase(const Char_t* data_type)
{
   // Similar to CleanMultiRunfiles(), but here the aim is only to remove from the
   // sqlite database any runfiles which are no longer physically present in the
   // repository i.e. they have already been deleted but our database was not updated.
 
   if (!fRepository) return;
 
   Info("CleanRunfileDataBase", "Checking for spurious duplicates (non-existent files)...");
 
   KVAvailableRunsFile* ARF = GetAvailableRunsFile(data_type);
 
   auto doubles = ARF->CheckMultiRunfiles();
   if (!doubles.empty()) {
      for (auto& ri : doubles) {
         // for each run/index, get full list of available files
         KVList dates, files;
         ARF->GetRunInfos(ri, dates, files);
         // now check that each file is physically present
         for (auto f : files) {
            TString fname = dynamic_cast<TObjString*>(f)->GetString();
            if (!fRepository->CheckFileStatus(this, data_type, fname)) {
               Info("CleanRunfileDataBase", "File %s does not exist : remove from database...", fname.Data());
               ARF->Remove(ri, fname);
            }
         }
      }
   }
   else
      Info("CleanRunfileDataBase", "Database is apparently consistent");
}
 
 
 
 
 
Bool_t KVDataSet::CheckRunfileUpToDate(const KVString& data_type, const run_index_t& run,
                                       KVDataSetRepository* other_repos)
{
   //Use this method to check whether the file of type "data_type" for run number "run"
   //in the data repository "other_repos" is more recent than the file contained in the data
   //repository corresponding to this dataset.
   //
   //Returns kFALSE if file in other repository is more recent.
 
   if (!other_repos)
      return kTRUE;
   //get dataset with same name as this one from dataset manager of other repository
   KVDataSet* ds = other_repos->GetDataSetManager()->GetDataSet(GetName());
   if (!ds) {
      KVError::Error(this, "CheckRunfileUpToDate",
            "Dataset \"%s\" not found in repository \"%s\"", GetName(),
            other_repos->GetName());
      return kFALSE;
   }
   //compare dates of the two runfiles
   if (GetRunfileDate(data_type, run) < ds->GetRunfileDate(data_type, run))
      return kFALSE;
   return kTRUE;
}
 
 
 
 
 
Bool_t KVDataSet::CheckRunfileUpToDate(const KVString& data_type, const run_index_t& run,
                                       const KVString& other_repos)
{
   //Use this method to check whether the file of type "data_type" for run number "run"
   //in the data repository "other_repos" is more recent than the file contained in the data
   //repository corresponding to this dataset.
   //Returns kTRUE if no repository with name "other_repos" exists.
   //Returns kFALSE if file in other repository is more recent.
 
   auto _or = gDataRepositoryManager->GetRepository(other_repos);
   if (_or)
      return CheckRunfileUpToDate(data_type, run, _or);
   KVError::Error(this, "CheckRunfileUpToDate",
         "No data repository known with this name : %s", (const char*) other_repos);
   return kTRUE;
}
 
 
 
 
 
void KVDataSet::CheckUpToDate(const Char_t* data_type,
                              const Char_t* other_repos)
{
   //Check whether all files of type "data_type" for run number "run" in the data repository
   //are up to date (i.e. at least as recent) as compared to the files in data repository "other_repos".
 
   if (!fRepository) return;
 
   auto _or =
      dynamic_cast<KVDataSetRepository*>(gDataRepositoryManager->GetRepository(other_repos));
   if (!_or) {
      KVError::Error(this, "CheckUpToDate",
            "No data repository known with this name : %s", other_repos);
      return;
   }
   auto runlist = GetAvailableRunsFile(data_type)->GetRunList();
   Int_t need_update = 0;
   for (auto& rr : runlist) {
      //check run
      if (!CheckRunfileUpToDate(data_type, rr, _or)) {
         cout << " *** run " << rr << " needs update ***" <<
              endl;
         cout << "\t\tREPOSITORY: " << fRepository->
              GetName() << "\tDATE: " << GetRunfileDate(data_type, rr).value().AsString() << endl;
         cout << "\t\tREPOSITORY: " << other_repos << "\tDATE: " << _or->
              GetDataSetManager()->GetDataSet(GetName())->
              GetRunfileDate(data_type, rr).value().AsString() << endl;
         need_update++;
      }
   }
   if (!need_update) {
      cout << " *** All runfiles are up to date for data type " <<
           data_type << endl;
   }
}
 
 
 
 
 
run_index_list KVDataSet::GetUpdatableRuns(const Char_t* data_type,
      const Char_t* other_repos)
{
   //Returns list of all runs of type "data_type" which may be updated
   //from the repository named "other_repos". See CheckUpToDate().
 
   run_index_list updates;
   if (!fRepository) return updates;
 
   auto _or = gDataRepositoryManager->GetRepository(other_repos);
   if (!_or) {
      KVError::Error(this, "CheckUpToDate",
            "No data repository known with this name : %s", other_repos);
      return updates;
   }
   auto runlist = GetAvailableRunsFile(data_type)->GetRunList();
   for (auto& rr : runlist) {
      //check run
      if (!CheckRunfileUpToDate(data_type, rr, _or)) {
         //run is out of date
         updates.Add(rr);
      }
   }
   return updates;
}
 
 
 
 
 
run_index_list KVDataSet::GetRunList(const Char_t* data_type,
                                     const KVDBSystem* system) const
{
   //Returns list of all files available for given "data_type"
   //
   //If a pointer to a reaction system is given, only files for the given system will be included in the list.
 
   if (!fRepository || !HasDataType(data_type)) {
      KVError::Error(this, "GetRunList",
            "No data of type %s available. Runlist will be empty.",
            data_type);
      return {};
   }
 
   return GetAvailableRunsFile(data_type)->GetRunList(system);
}
 
 
 
 
 
KVDataAnalysisTask* KVDataSet::GetAnalysisTask(const Char_t* keywords) const
{
   //This method returns a pointer to the available analysis task whose description (title) contains
   //all of the whitespace-separated keywords (which may be regular expressions)
   //given in the string "keywords". The comparison is case-insensitive.
   //
   //WARNING: this method can only be used to access analysis tasks that are
   //available for this dataset, i.e. for which the corresponding prerequisite data type
   //is available in the repository.
   //For unavailable data/tasks, use GetAnalysisTaskAny(const Char_t*).
   //
   //EXAMPLES
   //Let us suppose that the current dataset has the following list of tasks:
   //~~~
   // root [2] gDataSet->Print("tasks")
   //         1. Event reconstruction from raw data (raw->recon)
   //         2. Analysis of raw data
   //         3. Identification of reconstructed events (recon->ident)
   //         4. Analysis of reconstructed events (recon)
   //         5. Analysis of partially identified & calibrated reconstructed events (ident)
   //         6. Analysis of fully calibrated physical data (root)
   //~~~
   //Then the following will occur:
   //~~~
   // root [14] gDataSet->GetAnalysisTask("raw->recon")->Print()
   // KVDataAnalysisTask : Event reconstruction from raw data (raw->recon)
   //
   // root [10] gDataSet->GetAnalysisTask("analysis root")->Print()
   // KVDataAnalysisTask : Analysis of fully calibrated physical data (root)
   //~~~
 
   //case-insensitive search for matches in list based on 'title' attribute
   return (KVDataAnalysisTask*)fTasks.FindObjectAny("title", keywords, kTRUE, kFALSE);
}
 
 
 
 
 
bool KVDataSet::make_analysis_class(const KVDataAnalysisTask* dat, const Char_t* classname)
{
   if (!dat->WithUserClass()) {
      KVError::Error(this, "MakeAnalysisClass",
            "No user class required for analysis task \"%s\"", dat->GetTitle());
      return false;
   }
 
   //all analysis base classes must define a static Make(const Char_t * classname)
   //which generates the skeleton class files.
 
   TClass* cl = nullptr;
   //has the user base class for the task been compiled and loaded ?
   if (dat->CheckUserBaseClassIsLoaded()) cl = TClass::GetClass(dat->GetUserBaseClass());
   else
      return false;
 
   //set up call to static Make method
   unique_ptr<TMethodCall> methcall(new  TMethodCall(cl, "Make", Form("\"%s\"", classname)));
 
   if (!methcall->IsValid()) {
      KVError::Error(this, "MakeAnalysisClass", "static Make(const Char_t*) method for class %s is not valid",
            cl->GetName());
      return false;
   }
 
   //generate skeleton class
   methcall->Execute();
 
   return true;
}
 
 
 
 
TObject* KVDataSet::file_opener(const TString& type, const TString& fname, Option_t* opt)
{
   // For the give datatype 'type', open file using fullpath 'fname' and whatever kind of class is defined
   // to be used for opening files of this datatype. Returns file handle as a TObject* pointer.
   //
   //The default base classes for each type are defined as in this example:
   //
   // # Default base classes for reading runfiles of different types
   // DataSet.RunFileClass.raw:    KVRawDataReader
   // DataSet.RunFileClass.recon:    TFile
   // DataSet.RunFileClass.ident:    TFile
   // DataSet.RunFileClass.root:    TFile
   //
   //A different base class can be defined for a specific dataset/datatype
   //by adding a line to your $HOME/.kvrootrc like this:
   //
   //name_of_dataset.DataSet.RunFileClass.data_type:     BaseClassName
   //
   //The actual class to be used is then defined by plugins, for example
   //
   //name_of_dataset.DataSet.RunFileClass.data_type:     plugin_name
   //
   //where 'plugin_name' is one of the known plugins for the default base class for the given data_type,
   //i.e. if data_type="raw" we could use one of either "GANIL" or "MFM" as plugin_name:
   //
   //Plugin.KVRawDataReader:    GANIL    KVGANILDataReader     kaliveda-core    "KVGANILDataReader()"
   //+Plugin.KVRawDataReader:    MFM    KVMFMDataFileReader    kaliveda-core    "KVMFMDataFileReader()"
   //
   //If no plugin is found for the base class defined by DataSet.RunFileClass, the base class is used.
   //
   //To actually open the file, each base class & plugin must define a method
   //~~~{.cpp}
   //    static BaseClass* Open(const Char_t* path, Option_t* opt="", ...)
   //~~~
   //which takes the full path to the file as argument (any other arguments taking default options)
   //and returns a pointer of the BaseClass type to the created object which can be used to read the file.}
 
   auto plugin_class = GetDataSetEnv<KVString>(Form("DataSet.RunFileClass.%s", type.Data()));
 
   TPluginHandler* ph = nullptr;
   KVString base_class;
   if (IsThisAPlugin(plugin_class, base_class)) {
 
      //look for plugin specific to dataset & type
      ph = LoadPlugin(base_class, plugin_class);
 
   }
   else
      base_class = plugin_class;
   TClass* cl;
   if (!ph) {
      //no plugin - use base class
      cl = TClass::GetClass(base_class.Data());
   }
   else {
      cl = TClass::GetClass(ph->GetClass());
   }
 
   //set up call to static Open method
   std::unique_ptr<TMethodCall> methcall;
   if (strcmp(opt, "")) {
      //Open with option
      methcall = std::make_unique<TMethodCall>(cl, "Open", Form("\"%s\", \"%s\"", fname.Data(), opt));
   }
   else {
      //Open without option
      methcall = std::make_unique<TMethodCall>(cl, "Open", Form("\"%s\"", fname.Data()));
   }
 
   if (!methcall->IsValid()) {
      if (ph) KVError::Error(this, "OpenDataSetFile", "Open method for class %s is not valid", ph->GetClass());
      else KVError::Error(this, "OpenDataSetFile", "Open method for class %s is not valid", base_class.Data());
      return nullptr;
   }
 
   //open the file
   Long_t retval;
   methcall->Execute(retval);
   return ((TObject*)(retval));
}
 
 
 
 
bool KVDataSet::MakeAnalysisClass(const Char_t* task, const Char_t* classname)
{
   //Create a skeleton analysis class to be used for analysis of the data belonging to this dataset.
   //
   //  task = keywords contained in title of analysis task (see GetAnalysisTask(const Char_t*))
   //              (you do not need to include 'analysis', it is added automatically)
   //  classname = name of new analysis class
   //
   //Example:
   //  MakeAnalysisClass("raw", "MyRawDataAnalysis")
   //       -->  make skeleton raw data analysis class in files MyRawDataAnalysis.cpp & MyRawDataAnalysis.h
   //  MakeAnalysisClass("fully calibrated", "MyDataAnalysis")
   //       -->  make skeleton data analysis class in files MyDataAnalysis.cpp & MyDataAnalysis.h
 
   KVString _task = task;
   if (!_task.Contains("nalysis")) _task += " analysis";
   //We want to be able to write analysis classes even when we don't have any data
   //to analyse. Therefore we use GetAnalysisTaskAny.
   auto dat = GetAnalysisTaskAny(_task.Data());
   if (!dat) {
      KVError::Error(this, "MakeAnalysisClass",
            "called for unknown or unavailable analysis task : %s", _task.Data());
      return false;
   }
   return make_analysis_class(dat, classname);
}
 
 
 
 
bool KVDataSet::MakeAnalysisClass(int task, const Char_t* classname)
{
   //Create a skeleton analysis class to be used for analysis of the data belonging to this dataset.
   //
   //  task = index of analysis task as shown in KVDataSet::Print("tasks")
   //  classname = name of new analysis class
   //
 
   auto dat = GetAnalysisTask(task);
   if (!dat) {
      KVError::Error(this, "MakeAnalysisClass",
            "called for unknown or unavailable analysis task index : %d", task);
      return false;
   }
   return make_analysis_class(dat, classname);
}
 
 
 
 
 
Bool_t KVDataSet::OpenDataSetFile(const Char_t* filename, ifstream& file)
{
   // Look for (and open for reading, if found) the named file in the directory which
   // contains the files for this dataset (given by GetDataSetDir())
 
   return OpenDataSetFile(GetName(), filename, file);
}
 
 
 
 
Bool_t KVDataSet::OpenDataSetFile(const TString& dataset, const Char_t* filename, ifstream& file)
{
   // Static method to look for (and open for reading, if found) the named file in the directory which
   // contains the files for the dataset
 
   TString datasetdir = KVBase::GetDataSetEnv(dataset, "DataSet.Directory", dataset);
   if (gSystem->IsAbsoluteFileName(datasetdir)) {
      // dataset directory is outside of standard KV installation directories
      // use absolute path to search for file
      TString abspath;
      abspath.Form("%s/%s", datasetdir.Data(), filename);
      return SearchAndOpenKVFile(abspath, file);
   }
   // dataset directory is a subdirectory of GetDATADIRFilePath()
   return SearchAndOpenKVFile(filename, file, datasetdir);
}
 
 
 
 
TString KVDataSet::GetFullPathToDataSetFile(const Char_t* filename) const
{
   // Find a file in the dataset directory (given by GetDataSetDir())
   // Returns full path to file if found, empty string if not
 
   return GetFullPathToDataSetFile(GetName(), filename);
}
 
 
 
 
TString KVDataSet::GetFullPathToDataSetFile(const TString& dataset, const Char_t* filename)
{
   // Static method to find a file in the dataset directory (given by GetDataSetDir())
   //
   // Returns full path to file if found, empty string if not
 
   TString fullpath;
   TString datasetdir = KVBase::GetDataSetEnv(dataset, "DataSet.Directory", dataset);
   if (!SearchKVFile(filename, fullpath, datasetdir)) {
      KVError::Warning(KVDataSet{}, "GetFullPathToDataSetFile", "File %s not found in dataset subdirectory %s", filename, datasetdir.Data());
      fullpath = "";
   }
   return fullpath;
}
 
 
 
 
Bool_t KVDataSet::FindDataSetFile(const TString& dataset, const Char_t* filename)
{
   // Static method to find a file in the dataset directory (given by GetDataSetDir())
   // Returns kTRUE if found, kFALSE if not
 
   TString fullpath;
   TString datasetdir = KVBase::GetDataSetEnv(dataset, "DataSet.Directory", dataset);
   return SearchKVFile(filename, fullpath, datasetdir);
}
 
 
 
 
Bool_t KVDataSet::FindDataSetFile(const Char_t* filename) const
{
   // Find a file in the dataset directory (given by GetDataSetDir())
   // Returns kTRUE if found, kFALSE if not
 
   return FindDataSetFile(GetName(), filename);
}
 
 
 
 
 
KVDataAnalysisTask* KVDataSet::GetAnalysisTaskAny(const Char_t* keywords) const
{
   //This method returns the analysis task whose description (title) contains
   //all of the whitespace-separated keywords (which may be regular expressions)
   //given in the string "keywords". The comparison is case-insensitive.
   //The analysis task does not need to be "available", i.e. the associated prerequisite
   //data type does not have to be present in the repository (see GetAnalysisTask).
   //
   // If no task is found, returns nullptr
 
   KVDataAnalysisTask* tsk = (KVDataAnalysisTask*)fAllTasks.FindObjectAny("title", keywords, kTRUE, kFALSE);
   if (!tsk) {
      KVError::Error(this, "GetAnalysisTaskAny", "No task found with the following keywords in its title : %s", keywords);
   }
   return tsk;
}
 
 
 
 
 
Bool_t KVDataSet::DataBaseNeedsUpdate() const
{
   // Returns kTRUE if database needs to be regenerated from source files,
   // i.e. if source files are more recent than DataBase.root
   //
   // In case no directory exists for dataset (dataset added 'on the fly')
   // we create the directory and fill it with dummy files (Makefile, Runlist.csv, Systems.dat)
 
   TString pwd = gSystem->pwd();
 
   TString path = "";
   if (!SearchKVFile(GetDataSetDir(), path)) {
      // dataset directory doesn't exist - create it
      Info("DataBaseNeedsUpdate", "%s: Creating new dataset directory %s",
           GetName(), GetDataSetDir());
      if (gSystem->mkdir(GetDataSetDir())) {
         // problem creating directory
         KVError::Error(this, "DataBaseNeedsUpdate",
               "%s: Dataset directory %s does not exist and cannot be created ?",
               GetName(), GetDataSetDir());
         return kFALSE;
      }
      // create dummy files
      SearchKVFile(GetDataSetDir(), path); // get full path
      path += "/";
      TString filename = path + "Makefile";
      ofstream of1(filename.Data());
      of1 << "$(KV_WORK_DIR)/db/" << GetName() << "/DataBase.root : Runlist.csv Systems.dat" << endl;
      of1 << "\t@echo Database needs update" << endl;
      of1.close();
      filename = path + "Runlist.csv";
      ofstream of2(filename.Data());
      of2 << "# Automatically generated dummy Runlist.csv file" << endl;
      of2.close();
      filename = path + "Systems.dat";
      ofstream of3(filename.Data());
      of3 << "# Automatically generated dummy Systems.dat file" << endl;
      of3.close();
   }
   gSystem->cd(GetDataSetDir());
   TString cmd = "make -q";
   Int_t ret = gSystem->Exec(cmd.Data());
   gSystem->cd(pwd.Data());
   return (ret != 0);
}
 
 
 
 
TString KVDataSet::GetOutputRepository(const Char_t* taskname) const
{
   // Returns name of output repository for given task.
   // By default it is the name of the repository associated with this dataset,
   // but can be changed by the following environment variables:
   //
   // [repository].DefaultOutputRepository: [other repository]
   //    - this means that all tasks carried out on data in [repository]
   //      will have their output files placed in [other repository]
   //
   // [taskname].DataAnalysisTask.OutputRepository: [other repository]
   //    - this means that for [taskname], any output files will
   //      be placed in [other repository]
   //
   // [dataset].[taskname].DataAnalysisTask.OutputRepository: [other repository]
   //    - this means that for given [dataset] & [taskname],
   //      any output files will be placed in [other repository]
 
   if (gEnv->Defined(Form("%s.DataRepository.DefaultOutputRepository", GetRepository()->GetName())))
      return TString(gEnv->GetValue(Form("%s.DataRepository.DefaultOutputRepository", GetRepository()->GetName()), ""));
   TString orep = GetDataSetEnv(Form("%s.DataAnalysisTask.OutputRepository", taskname), GetRepository()->GetName());
   return orep;
}
 
 
 
 
 
void KVDataSet::CopyRunfilesFromRepository(const Char_t* type, const run_index_list& runs, const Char_t* destdir)
{
   // Copies the runfiles of given "type" into the local directory "destdir".
 
   auto repo = GetRepository();
   for (auto& run : runs) {
      TString filename = GetRunfileName(type, run);
      TString destpath;
      AssignAndDelete(destpath, gSystem->ConcatFileName(destdir, filename));
      repo->CopyFileFromRepository(this, type, filename, destpath);
   }
}
 
 
 
 
 
void KVDataSet::CopyRunfilesToRepository(const Char_t* type, const run_index_list& runs, const Char_t* destrepo)
{
   // Copies the runfiles of given "type" from the data repository associated
   // with this dataset into the local repository "destrepo".
 
   auto repo = GetRepository();
   auto dest_repo = dynamic_cast<KVDataSetRepository*>(gDataRepositoryManager->GetRepository(destrepo));
 
   if (!dest_repo) {
      KVError::Error(this, "CopyRunfilesToRepository", "Unknown destination repository : %s", destrepo);
      gDataRepositoryManager->Print();
      return;
   }
 
   KVDataSet* dest_ds = dest_repo->GetDataSetManager()->GetDataSet(GetName());
   dest_repo->CreateAllNeededSubdirectories(dest_ds, type);
   for (auto& run : runs) {
      TString filename = GetRunfileName(type, run);
      TString destpath = dest_repo->GetFullPathToTransferFile(dest_ds, type, filename);
      repo->CopyFileFromRepository(this, type, filename, destpath);
   }
}