KVAvailableRunsFile.cpp

/*
$Id: KVAvailableRunsFile.cpp,v 1.18 2009/03/12 14:01:02 franklan Exp $
$Revision: 1.18 $
$Date: 2009/03/12 14:01:02 $
*/
 
//Created by KVClassFactory on Fri May  5 10:46:40 2006
//Author: franklan
 
#include "KVAvailableRunsFile.h"
#include "KVDBRun.h"
#include "TObjArray.h"
#include "TObjString.h"
#include "KVString.h"
#include "KVList.h"
#include "KVDataRepository.h"
#include "KVRunFile.h"
 
KVString KVAvailableRunsFile::date_read_from_filename = "";
 
ClassImp(KVAvailableRunsFile)
 
 
 
std::optional<run_index_t> KVAvailableRunsFile::IsRunFileName(const Char_t* filename)
{
   // This method tests the string given as 'filename' to see if it could be the name of a runfile
   // of the datatype of this available runs file (GetDataType()), for its parent dataset, fDataSet.
   // Any protocol and/or path information in the filename is first removed.
   //
   // The format for a given dataset is defined by variable
   //
   // [dataset].DataSet.RunFileName.[datatype]:    [format]
 
   TString fmt = fDataSet->GetDataSetEnv(Form("DataSet.RunFileName.%s", GetDataType()), "");  //get format string for current dataset
   if (fmt == "") {
      Error("IsRunFileName", "No default format set for datatype: %s",
            GetDataType());
      return {};
   }
   return IsRunFileName(fmt, filename);
}
 
 
 
 
std::optional<run_index_t> KVAvailableRunsFile::IsRunFileName(const KVString& fmt, const KVString& filename, const KVString& separators)
{
   // This method tests the string given as 'filename' to see if it could be the name of a runfile.
   // Any protocol and/or path information in the filename is first removed.
   // The decision as to whether the filename is valid or not is based on whether it corresponds
   // to the given format string, e.g. run_%04d.dat
   //
   // If good, we return the run number & eventual index as a std::optional<run_index_t>
   // (if no index is defined for the file i.e. if %I does not appear in the format,
   // this part of the run_index_t has no value).
   //
   // If not, the std::optional has no value. This can be tested as:
   //
   //~~~{.cpp}
   // if(toto.IsRunFileName(...))
   // {
   //   ...ok, filename is good & run number was retrieved
   // }
   //~~~
   //
   // KNOWN CASES:
   //~~~
   // dataset         example filename                   format
   // =======         ================                   =====================
   // INDRA_camp1     run127.raw                         run%R.raw
   // INDRA_camp2     run127.raw                         run%R.raw
   // INDRA_camp4     run127.raw                         run%R.raw
   // INDRA_camp5     run127.raw                         run%R.raw
   // INDRA_e416a     Run356_06-Oct-05_12_44_12.dat      Run%R_%D.dat
   // INDRA_e475s     run_0085.dat.13-Mar-06.12:46:24    run_%R.dat.%D1.%D2
   // INDRA_e494s     run_0026.dat.04-May-07.20:33:00    run_%R.dat.%D1.%D2
   // INDRA_e503      run_0026.dat.04-May-07.20:33:00    run_%R.dat.%D1.%D2
   // INDRA_e613      run_0022.dat.08Sep11_00h54m47s     run_%R.dat.%D
   // INDRAFAZIA.E789 run_0017.dat.04-04-19_08h25m57s.1  run_%R.dat.%D.%I
   // FAZIA*          run000345                          run%R
   //
   //    %R      :   run number
   //    %D      :   date and time
   //    %D1.%D2 :   date and time separated by '.'
   //    %I      :   file index number. WARNING: the first file has NO index, the
   //                second file has index '1', etc. The index returned for the first
   //                file is zero.
   //~~~
   //
   // The filename will be broken up according to the separators (default: ".")
   // Then the different parts will be analysed according to the given format string
   KVString _file(gSystem->BaseName(filename));  //Remove protocol and/or path
 
   // the filename should contain the same number of "."-separated parts as the format
   // string, unless the format contains "%I", in which case it can have one less part
   // (case of first file for run with no index)
   int np_fmt = fmt.GetNValues(separators);
   bool with_index = fmt.Contains("%I");
   bool two_part_date = false;
   bool got_date = false;
   int np_fn = _file.GetNValues(separators);
   date_read_from_filename = "";
   KVString _fmt = fmt;
   if (np_fn == np_fmt + 1) {
      // if there is 1 more segment after splitting of the filename than in the format string,
      // try adding an index at the end
      _fmt.Append(".%I");
      with_index = true;
      ++np_fmt;
   }
   if (np_fmt == np_fn || (with_index && np_fn == np_fmt - 1)) {
      _file.Begin(separators);
      _fmt.Begin(separators);
      int index(-1), run(0);
      char date1[100], date2[100];
      if (fmt.Contains("%D1") && fmt.Contains("%D2")) two_part_date = true;
 
      while (!_file.End()) {
         KVString run_part = _file.Next();
         KVString fmt_part = _fmt.Next();
 
         if (fmt_part.Contains("%R")) {
            fmt_part.ReplaceAll("%R", "%d");
            if (fmt_part.Contains("%D")) {
               fmt_part.ReplaceAll("%D", "%s");
               sscanf(run_part.Data(), fmt_part.Data(), &run, date1);
               got_date = true;
            }
            else
               sscanf(run_part.Data(), fmt_part.Data(), &run);
         }
         else if (fmt_part.Contains("%I")) {
            fmt_part.ReplaceAll("%I", "%d");
            sscanf(run_part.Data(), fmt_part.Data(), &index);
         }
         else if (fmt_part.Contains("%D1")) {
            fmt_part.ReplaceAll("%D1", "%s");
            sscanf(run_part.Data(), fmt_part.Data(), date1);
            got_date = true;
         }
         else if (fmt_part.Contains("%D2")) {
            fmt_part.ReplaceAll("%D2", "%s");
            sscanf(run_part.Data(), fmt_part.Data(), date2);
            got_date = true;
         }
         else if (fmt_part.Contains("%D")) {
            fmt_part.ReplaceAll("%D", "%s");
            sscanf(run_part.Data(), fmt_part.Data(), date1);
            got_date = true;
         }
      }
      if (got_date) {
         if (two_part_date) date_read_from_filename.Form("%s.%s", date1, date2);
         else date_read_from_filename = date1;
      }
      if (with_index) return run_index_t{run, index};
      else return run_index_t{run, std::nullopt};
   }
   return {};
}
 
 
 
 
Bool_t KVAvailableRunsFile::ExtractDateFromFileName(const TString& fmt, const Char_t* name, KVDatime& date)
{
   // We assume that 'name' is the name of a runfile according to given format 'fmt'.
   // (see KVAvailableRunsFile::IsRunFileName for accepted formats for runfile names with dates).
   // We attempt several methods to try to extract a date from 'name'.
   // If successful, we return kTRUE and 'date' contains the result.
 
   auto run = IsRunFileName(fmt, name);
   if (!run) return kFALSE;
   // if runfile name contains a date, it is now in variable date_read_from_filename
   if (date_read_from_filename == "") return kFALSE;
   date_read_from_filename.ReplaceAll("_", " ");
   date_read_from_filename.ToUpper();
   if (KVDatime::IsSQLFormat(date_read_from_filename.Data())) {
      date.SetSQLDate(date_read_from_filename.Data());
      return kTRUE;
   }
   else {
      date_read_from_filename.ReplaceAll(".", " ");
      if (KVDatime::IsGANACQFormat(date_read_from_filename.Data())) {
         date.SetGanacqDate(date_read_from_filename.Data());
         return kTRUE;
      }
      else if (KVDatime::IsGANACQ2010Format(date_read_from_filename.Data())) {
         date.SetGanacq2010Date(date_read_from_filename.Data());
         return kTRUE;
      }
      else if (KVDatime::IsGANACQNarvalFormat(date_read_from_filename.Data())) {
         date.SetGanacqNarvalDate(date_read_from_filename.Data());
         return kTRUE;
      }
   }
   return kFALSE;
}
 
 
 
 
bool KVAvailableRunsFile::get_file_modification_date(KVDataRepository* repository, const KVBase* objs, KVDatime& modt)
{
   FileStat_t fs;
   if (repository->GetFileInfo(fDataSet, GetDataType(), objs->GetName(), fs)) {
      //runfile exists in repository
      modt = KVDatime(fs.fMtime);
      return kTRUE;
   }
   return kFALSE;
}
 
 
 
 
bool KVAvailableRunsFile::insert_runfile_into_database(KVDataRepository* repository,
      const KVBase* objs,
      DataBase* RunListTable,
      const run_index_t& run_num,
      KVDBRun* run
      , bool with_version_and_username)
{
   KVString filename = objs->GetName();
   KVDatime modt;
   if (get_file_modification_date(repository, objs, modt)) {
      insert_runfile_into_database(repository, filename, modt, RunListTable, run_num, run, with_version_and_username);
      return true;
   }
   return false;
}
 
 
 
 
void KVAvailableRunsFile::insert_runfile_into_database(KVDataRepository* repository, const KVString& filename, const KVDatime& modt, DataBase* RunListTable,
      const run_index_t& run_num, KVDBRun* run, bool with_version_and_username)
{
#ifdef WITH_RSQLITE
   auto& runs_table = RunListTable->get_table("Available_Runs");
   runs_table.prepare_insert_single_row();
   runs_table["run"] = run_num.run();
   if (run_num.has_index())
      runs_table["index"] = run_num.index();
   else
      runs_table["index"].set_null();
   runs_table["filename"] = filename;
   runs_table["date"] = modt.AsSQLString();
   int sys_id = 1;
   // is the run associated with a system which is already in the db?
   if (run && run->GetSystem()) {
      auto& sys_table = RunListTable->get_table("Systems");
      if (!RunListTable->count("Systems", "*", Form("system = '%s'", run->GetSystemName()))) {
         // system not in table; need to add it
         sys_table.prepare_insert_single_row();
         sys_table["system"] = run->GetSystemName();
         sys_table.insert_single_row();
      }
      // now retrieve system id for foreign key cross-reference
      RunListTable->select_data("Systems", "id", Form("system = '%s'", run->GetSystemName()));
      while (RunListTable->get_next_result())
         sys_id = sys_table["id"].get_data<int>();
   }
   runs_table["sys_id"] = sys_id;
   if (with_version_and_username) {
      std::unique_ptr<UserGroup_t> userinfo(gSystem->GetUserInfo());
      runs_table["username"] = userinfo->fUser;
      runs_table["version"] = GetKVVersion();
   }
   runs_table.insert_single_row();
#endif
}
 
 
 
 
void KVAvailableRunsFile::Update(const std::unique_ptr<DataBase>& db_handle)
{
   // Examine the contents of the repository directory corresponding to this datatype
   // for parent dataset fDataSet.
   //
   // For each file found which was not already in the list of available runs and
   // we add an entry to the available runlist database file.
   //
   // This may be called by OpenAvailableRunsFile() in order to initially create
   // the new db file: in this case db_handle contains the pointer to the (already open) db.
 
#ifdef WITH_RSQLITE
   KVSQLite::database* RunListTable;
   std::unique_ptr<KVSQLite::database> _db;
   if (db_handle) {
      // we have been called by OpenAvailableRunsFile() to create a new db file
      RunListTable = db_handle.get();
   }
   else {
      // we have been called to either create or update an existing db file
      _db = OpenAvailableRunsFile(true);
      RunListTable = _db.get();
   }
 
   auto& runs_table = RunListTable->get_table("Available_Runs");
   auto& sys_table = RunListTable->get_table("Systems");
 
   KVDataRepository* repository = fDataSet->GetRepository();
 
   std::cout << std::endl << "Updating runlist : " << std::flush;
   //get directory listing from repository
   auto dir_list = repository->GetDirectoryListing(fDataSet, GetDataType());
   if (!dir_list)
      return;
 
   TIter next(&dir_list);
   KVBase* objs;
   //progress bar
   Int_t ntot = dir_list.GetSize();
   Int_t n5pc = TMath::Max(ntot / 20, 1);
   Int_t ndone = 0;
   std::unique_ptr<KVExpDB> db_garbage;
   KVExpDB* db = fDataSet->GetDataBase("minimal");
   if (!db) {
      db = new KVExpDB();
      db_garbage.reset(db);//clean up
   }
   while ((objs = (KVBase*) next())) {      // loop over all files in directory
 
      auto run_num = IsRunFileName(objs->GetName());
 
      if (run_num) { // file has correct name format for runfile
 
         KVDBRun* run = db->GetDBRun(run_num.value().run());
 
         if (!run) {
            // if dataset has an index multiplier, this may be a recon file with an artificial run number
            if (db->HasIndexMultiplier()) {
               // try to decompose run number into run + index
               auto r = run_num.value().run() / db->GetIndexMultiplier();
               auto i = run_num.value().run() % db->GetIndexMultiplier();
               run_num->clear();
               run_num->set_run(r);
               if (i) run_num->set_index(i);
               run = db->GetDBRun(r);
            }
         }
 
         if (run && run->HasRunFile(run_num.value())) { // the run number & index is included in the database of runs
 
            bool add_to_db = true;
            if (!making_new_db) {
               // is there an entry for the same run/index & filename in the database?
               int occurences = 0;
               auto selection = SelectFileName(objs->GetName()) + " AND " + SelectRunAndOrIndex(run_num.value());
               occurences = RunListTable->count("Available_Runs", "*", selection);
               add_to_db = (occurences == 0);
            }
 
            if (add_to_db) {
               insert_runfile_into_database(repository, objs, RunListTable, run_num.value(), run);
            }
         }
         else { // run not included in database of runs
            Info("Update", "the current run [%s] is not in database", objs->GetName());
            if (!insert_runfile_into_database(repository, objs, RunListTable, run_num.value(), nullptr))
               Warning("Update", "%s GetFileInfo return kFALSE", objs->GetName());
         }
      }
 
      ndone++;
      if (!(ndone % n5pc))
         std::cout << '>' << std::flush;
   }
 
   std::cout << " DONE" << std::endl;
   making_new_db = false;
#endif
}
 
 
 
 
 
Bool_t KVAvailableRunsFile::GetRunInfo(const run_index_t& run, TDatime& modtime, TString& filename)
{
   // Look for a given run/index number in the DB, and read the file's modification date/time and filename
   //
   // \returns kTRUE if runfile found in DB, otherwise kFALSE
   //
   // If available runs file does not exist, Update() is called to create it.
   //
   // If there are multiple versions of the same run (abnormal situation), we print a warning and give info on the most recent file.
 
   KVList filenames, dates;
   //get infos for all versions of run
   GetRunInfos(run, dates, filenames);
   if (filenames.GetEntries() == 1) { //only one version
      filename = ((TObjString*) filenames.First())->String();
      modtime = ((TObjString*) dates.First())->String().Data();
      return kTRUE;
   }
   else if (filenames.GetEntries() > 1) {   //several versions
      Warning("GetRunInfo",
              "Multiple versions of this runfile exist in the repository. Infos for most recent file will be returned.");
      Warning("GetRunInfo",
              "You should clean the repository using KVDataSet::CleanMultiRunfiles.");
      //find most recent version
      TDatime most_recent("1998-12-25 00:00:00");
      Int_t i_most_recent = 0;
      for (int i = 0; i < dates.GetEntries(); i++) {
         //check if run is most recent
         TDatime rundate(((TObjString*) dates.At(i))->String().Data());
         if (rundate > most_recent) {
            most_recent = rundate;
            i_most_recent = i;
         }
      }
      filename = ((TObjString*) filenames.At(i_most_recent))->String();
      modtime = ((TObjString*) dates.At(i_most_recent))->String().Data();
      return kTRUE;
   }
   return kFALSE;
}
 
 
 
 
void KVAvailableRunsFile::GetRunInfos(const run_index_t& run, KVList& dates, KVList& files)
{
   //Look for a given run/index in the database, and store the file's modification date/time and filename
   //in the two KVList as TObjString objects (these objects belong to the
   //lists and will be deleted by them).
   //
   //We do not stop at the first run found, but add informations for every occurence of the run in the database
   //(however, we do ignore any spurious duplicate entries if they exist)
   //
   //If available runs db does not exist, it will be created
 
#ifdef WITH_RSQLITE
   auto RunDB = OpenAvailableRunsFile();
 
   //clear lists - delete objects
   dates.Clear();
   files.Clear();
 
   auto& runtable = RunDB->get_table("Available_Runs");
   if (RunDB->select_data("Available_Runs", "filename,date", SelectRunAndOrIndex(run), kTRUE)) {
      while (RunDB->get_next_result()) {
         dates.Add(new TObjString(runtable["date"].data().GetString()));
         files.Add(new TObjString(runtable["filename"].data().GetString()));
      }
   }
   else {
      Error("GetRunInfos", "DB query failure? selection=%s", SelectRunAndOrIndex(run).Data());
   }
#endif
}
 
 
#ifdef WITH_RSQLITE
 
 
std::vector<KVAvailableRunsFile::runfile_info_t> KVAvailableRunsFile::GetRunInfos(const run_index_t& run)
{
   // \return vector of runfile_info_t = std::pair<TDatime,TString> for each runfile in DB for given run_index_t
   auto RunDB = OpenAvailableRunsFile();
 
   std::vector<runfile_info_t> infos;
 
   auto& runtable = RunDB->get_table("Available_Runs");
   if (RunDB->select_data("Available_Runs", "filename,date", SelectRunAndOrIndex(run), kTRUE)) {
      while (RunDB->get_next_result()) {
         infos.push_back(std::make_pair(KVDatime(runtable["date"].data().GetString(), KVDatime::kSQL), TString(runtable["filename"].data().GetString())));
      }
   }
   else {
      Error("GetRunInfos", "DB query failure? selection=%s", SelectRunAndOrIndex(run).Data());
   }
   return infos;
}
 
 
 
 
std::map<run_index_t, std::vector<KVAvailableRunsFile::runfile_info_t>> KVAvailableRunsFile::GetRunFileInfos(const run_index_list& runlist)
{
   // For all runfiles in the run_index_list, fill the map with the dates & filenames from the DB
   std::map<run_index_t, std::vector<KVAvailableRunsFile::runfile_info_t>> infos;
   for (auto& ri : runlist) {
      infos[ri] = GetRunInfos(ri);
   }
   return infos;
}
 
 
 
 
void KVAvailableRunsFile::PrintRunFileInfos(const run_index_list& runlist)
{
   // For all runfiles in the run_index_list, print list of all filenames & dates from the DB.
 
   auto info_map = GetRunFileInfos(runlist);
   auto db = fDataSet->GetDataBase("minimal");
   KVDBSystem* sys = nullptr;
   for (auto& p : info_map) {
      auto S = db->GetDBRun(p.first.run())->GetSystem();
      if (S != sys) {
         std::cout << "\t" << S->GetName() << std::endl;
         sys = S;
      }
      std::cout << p.first << ":";
      for (auto& rf : p.second) {
         std::cout << "\t" << rf.first.AsSQLString() << "\t" << rf.second << std::endl;
      }
   }
}
 
#endif
 
 
 
 
KVUnownedList KVAvailableRunsFile::GetListOfAvailableSystems()
{
   //\return a sorted list of available systems based on the runs in the available runs database.
   //
   //For each system in the list we set the number of available runs : this number
   //can be retrieved with KVDBSystem::GetNumberRuns()
   //
   // \note 'bad' runfiles are excluded
 
   KVUnownedList sys_list;
 
   // we don't need a full database, just the systems & runs will do
   KVExpDB* db = fDataSet->GetDataBase("minimal");
   std::unique_ptr<KVExpDB> garbage_db;
   if (!db) {
      db = new KVExpDB;
      garbage_db.reset(db);//clean up
   }
#ifdef WITH_RSQLITE
   auto runDB = OpenAvailableRunsFile();
 
   // list of systems for available runs
   runDB->select_data("Systems", "system", "id > 1");
   while (runDB->get_next_result()) {
      auto sys = db->GetSystem((*runDB)["Systems"]["system"].get_data<TString>());
      if (sys && sys->GetNGoodRunFiles()) sys_list.Add(sys);
   }
   // set number of runs & events associated with each system
   TIter it_sys(&sys_list);
   while (auto sys = (KVDBSystem*)it_sys()) {
      runDB->select_data("Available_Runs,Systems", "run,index", Form("system = '%s'", sys->GetName()), true);
      std::map<int, int> nruns;
      ULong64_t events = 0;
      while (runDB->get_next_result()) {
         auto run = (*runDB)["Available_Runs"]["run"].get_data<int>();
         auto index = (*runDB)["Available_Runs"]["index"].is_null() ? -1 : (*runDB)["Available_Runs"]["index"].get_data<int>();
         if (!db->GetDBRunFile({run, index}).IsBad()) {
            ++nruns[run];
            events += db->GetDBRunFile({run, index}).GetEvents();
         }
      }
      sys->SetNumberRuns(nruns.size());
      sys->SetEvents(events);
   }
 
   //sort list of systems in order of increasing run number
   if (sys_list.GetSize() > 1)
      sys_list.Sort();
#endif
   return sys_list;
}
 
 
 
 
KVUniqueNameList KVAvailableRunsFile::GetSystemsForRunFiles(const run_index_list& rl) const
{
   // \return list of systems corresponding to given runfiles
   KVExpDB* db = fDataSet->GetDataBase("minimal");
   if (!db)
      return {};
   KVUniqueNameList syslist;
   for (auto& ri : rl) {
      KVDBRun* r{nullptr};
      KVDBSystem* s{nullptr};
      if ((r = db->GetDBRun(ri.run())) && (s = r->GetSystem()))
         syslist.Add(s);
   }
   return syslist;
}
 
 
 
 
KVList KVAvailableRunsFile::GetListOfAvailableRunFilesForSystem(const KVDBSystem* systol)
{
   //\param[in] systol pointer to system description in experimental database
   //
   //\return sorted list of available run files (KVRunFile objects) for the given system.
   //
   // \note 'bad' runfiles are excluded
 
   KVList sys_list;
 
   KVExpDB* db = fDataSet->GetDataBase("minimal");
   std::unique_ptr<KVExpDB> garbage_db;
   if (!db) {
      db = new KVExpDB;
      garbage_db.reset(db);//clean up
   }
#ifdef WITH_RSQLITE
   auto runDB = OpenAvailableRunsFile();
 
   // list of runfiles for given system
   runDB->select_data("Available_Runs,Systems", "run,index,filename,date,version,username", Form("system = '%s'", systol->GetName()), true);
   while (runDB->get_next_result()) {
      auto a_run = db->GetDBRun((*runDB)["Available_Runs"]["run"].get_data<int>());
      auto run = a_run->GetNumber();
      std::optional<int> index = ((*runDB)["Available_Runs"]["index"].is_null() ?
                                  std::nullopt :  std::optional<int>((*runDB)["Available_Runs"]["index"].get_data<int>()));
      KVDatime fDatime((*runDB)["Available_Runs"]["date"].get_data<TString>(), KVDatime::kSQL);
      KVString kvversion = (*runDB)["Available_Runs"]["version"].get_data<KVString>();
      KVString username = (*runDB)["Available_Runs"]["username"].get_data<KVString>();
      KVString filename = (*runDB)["Available_Runs"]["filename"].get_data<KVString>();
      if (IsCalled("raw")) {
         // for raw data files, use the actual date/time of the start of data taking in the database
         TString start_date = a_run->GetRunFile({run, index}).GetStartDate();
         if (start_date != "") fDatime.Set(start_date);
      }
      if (!db->GetDBRunFile({run, index}).IsBad())
         sys_list.Add(new KVRunFile(a_run, {run, index}, filename, fDatime, kvversion, username));
   }
 
   //sort list of runs in order of increasing run number
   if (sys_list.GetSize() > 1)
      sys_list.Sort();
#endif
   return sys_list;
}
 
 
 
 
void KVAvailableRunsFile::UpdateInfos(const run_index_t& run, const KVString& filename, const KVString& kvversion, const KVString& username)
{
   // Call this method to update informations on the file "filename" corresponding to run,
   // by adding/replacing the KV version and username read from the file itself (not necessarily
   // corresponding to current KV version and username)
#ifdef WITH_RSQLITE
   auto runDB = OpenAvailableRunsFile();
 
   auto selection = SelectRunAndOrIndex(run) + " AND " + SelectFileName(filename);
 
   (*runDB)["Available_Runs"]["version"] = kvversion;
   (*runDB)["Available_Runs"]["username"] = username;
   runDB->update("Available_Runs", "version,username", selection);
#endif
}
 
 
 
 
void KVAvailableRunsFile::Remove(const run_index_t& run, const KVString& filename)
{
   //Remove from the db ALL entries corresponding to the given run/index (and filename if given)
 
#ifdef WITH_RSQLITE
   auto runDB = OpenAvailableRunsFile();
 
   auto selection = SelectRunAndOrIndex(run);
   if (!filename.IsNull()) selection += " AND " + SelectFileName(filename);
 
   runDB->delete_rows_in_table("Available_Runs", selection);
#endif
}
 
 
 
 
 
void KVAvailableRunsFile::Add(const run_index_t& run, const KVString& filename)
{
   //Add to the file an entry corresponding to this run, assumed to be present in the repository
   //with the given filename.
 
#ifdef WITH_RSQLITE
   auto runDB = OpenAvailableRunsFile();
 
   insert_runfile_into_database(fDataSet->GetRepository(), filename, KVDatime(), runDB.get(), run, fDataSet->GetDataBase("minimal")->GetDBRun(run.run()), true);
#endif
}
 
 
 
 
std::forward_list<run_index_t> KVAvailableRunsFile::CheckMultiRunfiles()
{
   //Returns a list with all run/indexes for which more than one file is in the available runs db
 
   std::forward_list<run_index_t> multiruns;
#ifdef WITH_RSQLITE
   auto runDB = OpenAvailableRunsFile();
 
   runDB->select_data("Available_Runs", "run,index,COUNT(*)", "", false, "GROUP BY run, index HAVING COUNT(*)>1");
 
   while (runDB->get_next_result()) {
      multiruns.push_front({(*runDB)["Available_Runs"]["run"].get_data<int>(),
                            ((*runDB)["Available_Runs"]["index"].is_null() ?
                             std::nullopt :  std::optional<int>((*runDB)["Available_Runs"]["index"].get_data<int>()))});
   }
#endif
   return multiruns;
}
 
 
 
 
 
run_index_list KVAvailableRunsFile::GetRunList(const KVDBSystem* sys)
{
   //Returns list of available run/indexes for this data type.
   //
   //If 'sys' gives the address of a valid database reaction system, only files
   //corresponding to the system will be included.
   //
   //Note that runfiles which are considered to be 'bad' are excluded
 
   TString selection, tables("Available_Runs");
   if (sys) {
      selection = Form("system = '%s'", sys->GetName());
      tables += ",Systems";
   }
 
   KVExpDB* db = fDataSet->GetDataBase("minimal");
   std::unique_ptr<KVExpDB> garbage_db;
   if (!db) {
      db = new KVExpDB;
      garbage_db.reset(db);//clean up
   }
   run_index_list l;
#ifdef WITH_RSQLITE
   auto runDB = OpenAvailableRunsFile();
 
   runDB->select_data(tables, "run,index", selection);
   while (runDB->get_next_result()) {
      auto run = (*runDB)["Available_Runs"]["run"].get_data<int>();
      auto index = (*runDB)["Available_Runs"]["index"].is_null() ? -1 : (*runDB)["Available_Runs"]["index"].get_data<int>();
      if (!db->GetDBRunFile({run, index}).IsBad())
         l.Add({run, index});
   }
#endif
   return l;
}
 
 
 
 
KVString KVAvailableRunsFile::GetFileName() const
{
   // Filename of file containing information on available runs
   // i.e. [repository].available_runs.[dataset subdir].[dattype subdir].sqlite
   //
   // If [dataset subdir] contains "/" (i.e. if data is grouped into subdirectories)
   // we replace them by "_"
 
   KVString filename;
   if (!fDataSet) {
      Error("GetFileName", "Dataset has not been set for this file.");
      filename = "";
   }
   else {
      KVString datapath = fDataSet->GetDataPathSubdir();
      datapath.ReplaceAll("/", "_");
      filename.Form("%s.available_runs.%s.%s.sqlite", fDataSet->GetRepository()->GetName(), datapath.Data(),
                    fDataSet->GetDataTypeSubdir(GetDataType()).Data());
   }
   return filename;
}
 
 
 
 
KVString KVAvailableRunsFile::GetFilePath() const
{
   // Returns the full path to the directory where the available runs file is stored.
   // This is by default the same directory where the dataset files are stored.
   // However, if KaliVeda was installed using a GNU-style directory layout
   // (possibly in the system directories, e.g. Ubuntu packages)
   // then the path will be the user's working directory + dataset name
   //
   // Alternatively, by setting the config variable
   //   KVAvailableRunsFile.RunFileDirectory:   /some/path
   //
   // files will be stored/looked for in
   //   /some/path/[dataset]
   //
   // You can use environment variables - $(VAR) - in the path
 
   KVString filepath;
   if (!fDataSet) {
      Error("GetFilePath", "Dataset has not been set for this file.");
      filepath = "";
   }
   else {
      TString p = gEnv->GetValue("KVAvailableRunsFile.RunFileDirectory", "");
      if (p == "") {
         // no user-defined run file directory
         if (is_gnuinstall()) {
            // GNU-style install: use working directory
            filepath = GetWORKDIRFilePath(fDataSet->GetName());
         }
         else
            filepath = fDataSet->GetDataSetDir();
      }
      else {
         gSystem->ExpandPathName(p);
         AssignAndDelete(filepath, gSystem->ConcatFileName(p.Data(), fDataSet->GetName()));
      }
   }
   return filepath;
}
 
 
 
 
KVString KVAvailableRunsFile::GetFullPathToAvailableRunsFile() const
{
   // Return the full path on the local machine to the file
   // used to store information on available runs
 
   KVString path;
   AssignAndDelete(path, gSystem->ConcatFileName(GetFilePath(), GetFileName()));
   return path;
}
 
 
 
 
Bool_t KVAvailableRunsFile::CheckDirectoryForAvailableRunsFile()
{
   // Check that the directory given by GetFilePath() exists and is writable
   // If it does not exist, it will be created if possible
   // If it exists but is not writable, or cannot be created, returns kFALSE.
 
   if (gSystem->AccessPathName(GetFilePath(), kFileExists)) { // directory does not exist
      if (gSystem->mkdir(GetFilePath(), kTRUE) == -1) {
         Error("CheckDirectoryForAvailableRunsFile", "cannot create directory %s - check access permissions", GetFilePath().Data());
         return kFALSE;
      }
      // set access permissions on created directory
      gSystem->Chmod(GetFilePath(), 0775);
   }
   if (gSystem->AccessPathName(GetFilePath(), kWritePermission)) { // directory is not writable
      Error("CheckDirectoryForAvailableRunsFile", "directory %s is not writable - check access permissions", GetFilePath().Data());
      return kFALSE;
   }
   return kTRUE;
}
 
 
 
 
std::unique_ptr<DataBase> KVAvailableRunsFile::OpenAvailableRunsFile(bool called_by_update)
{
#ifdef WITH_RSQLITE
   CheckDirectoryForAvailableRunsFile();
   making_new_db = !FileExists();
   auto db = std::make_unique<KVSQLite::database>(GetFullPathToAvailableRunsFile());
   if (!db->good()) {
      Error("OpenAvailableRunsFile", "Failed to open/create sqlite database file %s", GetFullPathToAvailableRunsFile().Data());
      db.reset(nullptr);
      return db;
   }
   if (making_new_db) {
      // set access permissions on database file
      gSystem->Chmod(GetFullPathToAvailableRunsFile(), 0664);
 
      // create systems table
      KVSQLite::table sys_tab("Systems");
      sys_tab.add_column("id", KVSQLite::column_type::INTEGER);
      sys_tab.primary_key("id");
      sys_tab.add_column("system", KVSQLite::column_type::TEXT).NOT_NULL().UNIQUE();
      auto& t = db->add_table(sys_tab);
      // add default 'unknown' system (which will have id=1)
      t.prepare_insert_single_row();
      t["system"] = "unknown";
      t.insert_single_row();
 
      // create runs table
      KVSQLite::table run_tab("Available_Runs");
      run_tab.add_column("id", KVSQLite::column_type::INTEGER);
      run_tab.primary_key("id");
      run_tab.add_column("run", KVSQLite::column_type::INTEGER).NOT_NULL();
      run_tab.add_column("index", KVSQLite::column_type::INTEGER);
      run_tab.add_column("filename", KVSQLite::column_type::TEXT).NOT_NULL();
      run_tab.add_column("date", KVSQLite::column_type::TEXT).NOT_NULL();
      run_tab.add_column("version", KVSQLite::column_type::TEXT);
      run_tab.add_column("username", KVSQLite::column_type::TEXT);
      run_tab.add_column("sys_id", KVSQLite::column_type::INTEGER).NOT_NULL();
      run_tab.foreign_key("sys_id", "Systems", "id");
 
      db->add_table(run_tab);
 
      if (!called_by_update) Update(db);
   }
   return db;
#else
   return {};
#endif
}
 
 
 
 
Bool_t KVAvailableRunsFile::InfosNeedUpdate(const run_index_t& run, const KVString& filename)
{
   // return kTRUE if the given file for this runfile is lacking some information
   // e.g. the KV version and username
   //
   // N.B.: if no file is known for this run, we return kFALSE
#ifdef WITH_RSQLITE
   auto runDB = OpenAvailableRunsFile();
   auto selection = SelectRunAndOrIndex(run) + " AND " + SelectFileName(filename);
   runDB->select_data("Available_Runs", "run,version", selection);
   while (runDB->get_next_result()) {
      if ((*runDB)["Available_Runs"]["version"].is_null()) return true;
   }
#endif
   return false;
}