KaliVeda
Toolkit for HIC analysis
KVDataSet.cpp
1 /*
2 $Id: KVDataSet.cpp,v 1.41 2009/03/11 14:22:41 franklan Exp $
3 $Revision: 1.41 $
4 $Date: 2009/03/11 14:22:41 $
5 $Author: franklan $
6 */
7 
8 #include "TMethodCall.h"
9 #include "KVDataSet.h"
10 #include "KVDataRepository.h"
11 #include "KVDataRepositoryManager.h"
12 #include "KVDataSetManager.h"
13 #include "TSystem.h"
14 #include "TObjArray.h"
15 #include "TObjString.h"
16 #include "KVDBSystem.h"
17 #include "KVDBRun.h"
18 #include "TEnv.h"
19 #include "KVAvailableRunsFile.h"
20 #include "KVNumberList.h"
21 #include "TPluginManager.h"
22 #include "TClass.h"
23 #include "KVRunFile.h"
24 
25 using namespace std;
26 
28 
29 KVDataSet* gDataSet;
30 
31 
34 
36 {
37  //Default constructor
38  fRepository = nullptr;
39  fDataBase = nullptr;
40  fAllTasks.SetOwner(kTRUE);
41  fTasks.SetOwner(kFALSE);
42 }
43 
44 
45 
50 
52 {
53  // \param type name of data type ('raw', 'recon', ...)
54  // \returns pointer to available runs file object for given data type
55  // \note if no data repository is associated with dataset, returns nullptr
56  if (!fRepository) return nullptr;
57  KVAvailableRunsFile* avrf =
58  (KVAvailableRunsFile*) fAvailableRuns.FindObjectByName(type);
59  if (!avrf) {
60  avrf = fRepository->NewAvailableRunsFile(type, this);
61  fAvailableRuns.Add(avrf);
62  }
63  return avrf;
64 }
65 
66 
67 
82 
84 {
85  //Returns name of file containing database for dataset.
86  //
87  //This is fixed as `DataBase.root.M.mm` where
88  // - `M` is the current major version number
89  // - `m` is the current minor version number
90  //
91  //(fixed to be consistent with CMake function INSTALL_KALIVEDA_DATASETS which sets up
92  //the Makefile for each dataset which automatically triggers rebuilding of the ROOT file
93  //when source files change: thus we must have the right file name!)
94  //
95  //This in order to avoid problems if several different versions of KaliVeda
96  //use the same working directory (in user's `$HOME/.kaliveda`) to write the
97  //database files, which are often incompatible between versions
98 
99  TString n = "DataBase.root";
100  n.Append(Form(".%d.%d", GetKVMajorVersion(),
101  GetKVMinorVersion()));
102  return n;
103 }
104 
105 
106 
111 
113 {
114  //\returns name of database object in database file.
115  //
116  //If this is not set explicitly with SetDBName(), we use the name of the dataset by default
117 
118  return (fDBName != "" ? fDBName.Data() : GetName());
119 }
120 
121 
122 
123 
126 
127 void KVDataSet::OpenDBFile(const Char_t* full_path_to_dbfile) const
128 {
129  //Open the database from a file on disk.
130 
131  TDirectory* work_dir = gDirectory; //keep pointer to current directory
132  fDBase.reset(new TFile(full_path_to_dbfile, "READ"));
133 
134  if (fDBase->IsOpen()) {
135  fDataBase = dynamic_cast<KVExpDB*>(fDBase->Get(GetDBName()));
136  if (!fDataBase) {
137  Error("OpenDBFile", "%s not found in file %s", GetDBName().Data(),
138  GetDBFileName().Data());
139  }
140  else {
141  fDataBase->ReadObjects(fDBase.get()); // read any associated objects
142  }
143  work_dir->cd(); //back to initial working directory
144  }
145 }
146 
147 
148 
149 
152 
154 {
155  // \returns full path to file where database is written on disk
156 
157  TString dbfile = GetDBFileName();
158  TString dbfile_fullpath;
159  TString tmp;
160 
161  // If this dataset is just an alias for another dataset i.e. if DataSet.Directory
162  // is set with just the name of another dataset (not a full path to dataset files)
163  // then the database file should be written/found under the name of the alias.
164  TString dataset_alias = GetDataSetEnv("DataSet.Directory", GetName());
165  TString db_alias = GetName();
166  if (!gSystem->IsAbsoluteFileName(dataset_alias)) db_alias = dataset_alias;
167 
168  AssignAndDelete(tmp, gSystem->ConcatFileName(GetDATABASEFilePath(), db_alias.Data()));
169  AssignAndDelete(dbfile_fullpath, gSystem->ConcatFileName(tmp.Data(), dbfile.Data()));
170  return dbfile_fullpath;
171 }
172 
173 
174 
177 
179 {
180  //Check configuration variables to see if the task parameters have been "tweaked" for the dataset.
181 
182  KVString envar = GetDataSetEnv(Form("%s.DataAnalysisTask.Title", t->GetName()));
183  if (envar != "") t->SetTitle(envar);
184  envar = GetDataSetEnv(Form("%s.DataAnalysisTask.Analyser", t->GetName()));
185  if (envar != "") t->SetDataAnalyser(envar);
186  envar = GetDataSetEnv(Form("%s.DataAnalysisTask.UserClass.Base", t->GetName()));
187  if (envar != "") t->SetUserBaseClass(envar);
188  envar = GetDataSetEnv(Form("%s.DataAnalysisTask.Prereq", t->GetName()));
189  if (envar != "") t->SetPrereq(envar);
190  envar = GetDataSetEnv(Form("%s.DataAnalysisTask.UserClass.ExtraACliCIncludes", t->GetName()));
191  if (envar != "") t->SetExtraAClicIncludes(envar);
192  Int_t nev = (Int_t)GetDataSetEnv(Form("%s.DataAnalysisTask.StatusUpdateInterval", t->GetName()), 0.0);
193  if (nev > 0) t->SetStatusUpdateInterval(nev);
194 }
195 
196 
197 
198 
202 
204 {
205  // Write the experiment database for this dataset to disk (ROOT file), creating and setting
206  // permissions for any required directories
207 
208  TString dbfile_fullpath = GetFullPathToDB();
209  TString tmp = gSystem->DirName(dbfile_fullpath.Data()); //full path to directory $KVROOT/db/[dataset name]
210 
211  if (gSystem->AccessPathName(tmp.Data())) { // directory $KVROOT/db/[dataset name] does not exist
212 
213  if (gSystem->mkdir(tmp.Data()) == -1) { // problem creating $KVROOT/db/[dataset name]
214 
215  TString tmp2 = gSystem->DirName(tmp.Data());// full path to directory $KVROOT/db
216 
217  if (gSystem->AccessPathName(tmp2.Data())) { // directory $KVROOT/db does not exist
218 
219  if (gSystem->mkdir(tmp2.Data()) == -1) { // problem creating $KVROOT/db
220  Error("SaveDataBase", "Cannot create directory %s required to save database",
221  tmp2.Data());
222  return;
223  }
224  gSystem->Chmod(tmp2.Data(), 0775);
225  }
226  else {
227  Error("SaveDataBase", "Cannot create directory %s required to save database, even though %s exists: check disk space ?",
228  tmp.Data(), tmp2.Data());
229  return;
230  }
231  //try again
232  if (gSystem->mkdir(tmp.Data()) == -1) {
233  Error("SaveDataBase", "Cannot create directory %s required to save database",
234  tmp.Data());
235  return;
236  }
237  else {
238  gSystem->Chmod(tmp.Data(), 0775);
239  }
240  }
241  else {
242  gSystem->Chmod(tmp.Data(), 0775);
243  }
244  }
245 
246  WriteDBFile(dbfile_fullpath.Data());
247 }
248 
249 
250 
251 
254 
255 void KVDataSet::WriteDBFile(const Char_t* full_path_to_dbfile) const
256 {
257  //Write the database to disk.
258 
259  TDirectory* work_dir = gDirectory; //keep pointer to current directory
260  if (!fDataBase) {
261  Error("WriteDBFile", "Database has not been built");
262  return;
263  }
264  fDBase.reset(new TFile(full_path_to_dbfile, "recreate"));
265  fDBase->cd(); //set as current directory (maybe not necessary)
266  fDataBase->Write(GetDBName()); //write database to file with given name
267  fDataBase->WriteObjects(fDBase.get()); //write any associated objects
268  fDBase->Write(); // write file header etc.
269  fDBase->Close(); // close file
270  gSystem->Chmod(full_path_to_dbfile, 0664); // set permissions to rw-rw-r--
271  work_dir->cd(); //back to initial working directory
272 }
273 
274 
275 
284 
286 {
287  //\returns pointer to database associated with this dataset.
288  //
289  //Opens, updates or creates database file if necessary
290  //(the database is automatically rebuilt if the source files are
291  //more recent than the last database file).
292  //
293  //\param[in] opt if opt="update": force regeneration of the database from source files in dataset directory
294 
295  TString _opt(opt);
296  _opt.ToUpper();
297  if (_opt == "UPDATE") {
298  OpenDataBase(_opt.Data());
299  }
300  else {
301  OpenDataBase();
302  }
303  return fDataBase;
304 }
305 
306 
307 
318 
320 {
321  //Open the database for this dataset.
322  //
323  //If the database does not exist or is older than the source files
324  //the database is automatically rebuilt (see DataBaseNeedUpdate()).
325  //
326  //\param[in] opt if opt="UPDATE" we force rebuilding of the database
327  //
328  //\warning if the database needs to be (re)built, we set gDataSet to point to this dataset in case it was not already done,
329  //as in order to (re)build the database it may be necessary for gDataSet to point to the current dataset.
330 
331  Bool_t is_glob_db = kFALSE;
332  //if option="update" or database out of date or does not exist, (re)build the database
333  if ((!strcmp(opt, "UPDATE")) || DataBaseNeedsUpdate()) {
334  //check if it is the currently active database (gDataBase),
335  //in which case we must 'cd()' to it after rebuilding
336  Info("OpenDataBase", "Updating database file");
337  fDataBaseUpdateInProgress = true;
338  is_glob_db = (fDataBase == gExpDB);
339  if (fDataBase) {
340  delete fDataBase;
341  fDataBase = 0;
342  }
343  // make sure gDataSet is set & points to us
344  gDataSet = const_cast<KVDataSet*>(this);
345  fDataBase = KVExpDB::MakeDataBase(GetDBName(), GetDataSetDir());
346  if (!fDataBase) {
347  // no database defined for dataset
348  Info("OpenDataBase", "No database defined for dataset");
349  return;
350  }
351  SaveDataBase();
352  if (fDataBase && is_glob_db) fDataBase->cd();
353  fDataBaseUpdateInProgress = false;
354  }
355  else if (!fDataBase) {
356  // if database is not in memory at this point, we need to
357  // open the database file and read in the database
358 
359  //load plugin for database
360  if (!LoadPlugin("KVExpDB", GetDBName())) {
361  Error("GetDataBase", "Cannot load required plugin library");
362  return;
363  }
364  //look for database file in dataset subdirectory
365  TString dbfile_fullpath = GetFullPathToDB();
366  //open database file
367  OpenDBFile(dbfile_fullpath.Data());
368  }
369 }
370 
371 
372 
373 
376 
378 {
379  //Print dataset information
380  cout << "Dataset name=" << GetName() << " (" << GetTitle() << ")";
381  if (IsAvailable()) {
382  cout << " [ AVAILABLE: ";
383  cout << fDatatypes.Data();
384  cout << "]";
385  }
386  else
387  cout << " [UNAVAILABLE]";
388  cout << endl;
389 }
390 
391 
392 
399 
400 void KVDataSet::Print(Option_t* opt) const
401 {
402  //Print dataset information
403  //
404  //param[in] opt select optional output formats:
405  // - if string contains "tasks", print numbered list of tasks that can be performed
406  // - if string contains "data", print list of available data types
407 
408  TString Sopt(opt);
409  Sopt.ToUpper();
410  if (Sopt.Contains("TASK")) {
411  if (!GetNtasks()) {
412  cout << " *** No available analysis tasks ***"
413  << endl;
414  return;
415  }
416  else {
417  for (int i = 1; i <= GetNtasks(); i++) {
418  KVDataAnalysisTask* dat = GetAnalysisTask(i);
419  cout << "\t" << i << ". " << dat->GetTitle() << endl;
420  }
421  }
422  cout << endl;
423  }
424  else if (Sopt.Contains("DATA")) {
425  cout << "Available data types: " << fDatatypes.Data() << endl;
426  }
427  else {
428  ls(opt);
429  }
430 }
431 
432 
433 
442 
444 {
445  //Check if this data set is physically present and available for analysis.
446  //
447  //In other words we check if the value of GetDataPathSubdir() is a subdirectory
448  //of the current data repository.
449  //If so, we proceed to check for the existence of sudirectories corresponding
450  // to any of the datatypes associated with the dataset.
451  //
452 
453  if (!fRepository) // for a stand-alone KVDataSetManager not linked to a KVDataRepository,
454  SetAvailable(); // all known datasets are 'available'
455  else
456  SetAvailable(fRepository->CheckSubdirExists(GetDataPathSubdir()));
457  if (!IsAvailable())
458  return;
459  //check subdirectories
460  KVString data_types = GetDataSetEnv("KVDataSet.DataTypes", "");
461  if (data_types == "") {
462  Warning("CheckAvailable", "No datatypes defined for this dataset: %s\nCheck value of KVDataSet.DataTypes or %s.KVDataSet.DataTypes",
463  GetName(), GetName());
464  SetAvailable(kFALSE);
465  }
466  fDatatypes = "";
467  // loop over data types
468  data_types.Begin(" ");
469  while (!data_types.End()) {
470  KVString type = data_types.Next(kTRUE);
471  if (!fRepository ||
472  (fRepository && fRepository->CheckSubdirExists(GetDataPathSubdir(), GetDataTypeSubdir(type.Data())))
473  ) {
474  AddAvailableDataType(type.Data());
475  }
476  }
477  //check at least one datatype exists
478  SetAvailable(fDatatypes != "");
479  //check user name against allowed groups
480  if (!CheckUserCanAccess()) {
481  SetAvailable(kFALSE);
482  return;
483  }
484 }
485 
486 
487 
489 
491 {
492  if (fDatatypes != "") fDatatypes += " ";
493  KVString _type = type;
494  _type.Remove(TString::kBoth, ' '); //strip whitespace
495  fDatatypes += _type;
496 }
497 
498 
499 
506 
508 {
509  // Add to fAllTasks list any data analysis task in list 'task_list'
510  //
511  // Add to fTasks list any data analysis task in list 'task_list' whose pre-requisite datatype is present for this dataset.
512  //
513  // Any dataset-specific "tweaking" of the task (including the prerequisite datatype) is done here.
514 
515  TString availables = gEnv->GetValue(Form("%s.DataAnalysisTask", GetName()), "");
516  fAllTasks.Clear();
517  fTasks.Clear();
518  TIter nxt(task_list);
519  KVDataAnalysisTask* dat;
520  while ((dat = (KVDataAnalysisTask*) nxt())) {
521  //make new copy of default analysis task
522  if (availables == "" || availables.Contains(dat->GetName())) {
523  KVDataAnalysisTask* new_task = new KVDataAnalysisTask(*dat);
524  //check if any dataset-specific parameters need to be changed
525  SetDataSetSpecificTaskParameters(new_task);
526  fAllTasks.Add(new_task);
527  // add tasks with available prerequisite data to fTasks
528  if (HasDataType(new_task->GetPrereq())) {
529  fTasks.Add(new_task);
530  }
531  }
532  }
533 }
534 
535 
536 
540 
542 {
543  //Returns the number of tasks associated to dataset which are compatible
544  //with the available data
545 
546  return fTasks.GetSize();
547 }
548 
549 
550 
555 
557 {
558  //Return kth analysis task in list of available tasks.
559  //
560  //\param[in] k task number in range [1, GetNtasks()]. Corresponds to the number shown next to the title of the task when Print("tasks") is called
561  return (KVDataAnalysisTask*) fTasks.At(k - 1);
562 }
563 
564 
565 
572 
573 std::unique_ptr<TList> KVDataSet::GetListOfAvailableSystems(const Char_t* datatype, KVDBSystem* systol)
574 {
575  //\returns list of available systems or runfiles for this dataset and the given datatype
576  //
577  //\param[in] systol when systol=nullptr (default), return list of available systems. if pointer to system, return list of available runfiles for the given system
578  //
579  //If no systems are defined for the dataset then we return a list of available runfiles for the given datatype
580 
581  if (!GetAvailableRunsFile(datatype)) {
582  Error("GetListOfAvailableSystems(const Char_t*)",
583  "No available runs file for type %s", datatype);
584  return 0;
585  }
586  return GetAvailableRunsFile(datatype)->GetListOfAvailableSystems(systol);
587 }
588 
589 
590 
597 
599 {
600  //\returns list of available systems or runfiles for this dataset and the prerequisite datatype for the given analysis task
601  //
602  //\param[in] systol when systol=nullptr (default), return list of available systems. if pointer to system, return list of available runfiles for the given system
603  //
604  //If no systems are defined for the dataset then we return a list of available runfiles for the given datatype
605 
606  return GetListOfAvailableSystems(datan->GetPrereq(), systol);
607 }
608 
609 
610 
631 
632 void KVDataSet::SetName(const char* name)
633 {
634  // Set name of dataset
635  //
636  // Also sets path to directory containing database informations
637  // for this dataset, i.e. list of runs, systems, calibration files etc.
638  //
639  // By default, just the name of the dataset is used, i.e.
640  // `[DATADIR]/name`
641  // (where `DATADIR` = path given by KVBase::GetDATADIRFilePath())
642  //
643  // However, if the variable
644  //~~~
645  // [name].DataSet.Directory: [path]
646  //~~~
647  // has been set, the value of `[path]` will be used:
648  // - if [path] is an absolute path name, it will be used as such
649  // - if [path] is an incomplete or relative path, it will be prepended with `[DATADIR]/`
650  //
651  // This allows to use one dataset as an alias for another, by setting `DataSet.Directory`
652  // to the name of an existing dataset
654  TString path = GetDataSetEnv("DataSet.Directory", name);
655  if (gSystem->IsAbsoluteFileName(path)) fCalibDir = path;
656  else {
657  // in this case (not an absolute path but just the name of another dataset)
658  // this dataset is an alias for another dataset.
659  fCalibDir = GetDATADIRFilePath(path);
660  // the name of the database object is the name of the "true" dataset
661  SetDBName(path);
662  }
663 }
664 
665 
666 
669 
671 {
672  //\returns full path to directory containing database and calibration/identification parameters etc. for this dataset.
673 
674  return fCalibDir.Data();
675 }
676 
677 
678 
691 
692 void KVDataSet::cd() const
693 {
694  // Makes this dataset the "currently active" or default dataset.
695  //
696  //At the same time, the database and data repository associated with
697  //this dataset also become the "currently active" ones:
698  //
699  // | global pointer | represents | base class |
700  // |----------------|------------|------------|
701  // | `gDataSet` | active dataset | KVDataSet |
702  // | `gExpDB` | associated experimental database | KVExpDB |
703  // | `gDataRepository` | repository containing runfiles | KVDataRepository |
704  //
705 
706  gDataSet = const_cast<KVDataSet*>(this);
707  if (fRepository) fRepository->cd();
708  KVExpDB* db = GetDataBase();
709  if (db) db->cd();
710 }
711 
712 
713 
741 
743 {
744  // Open file containing data of given datatype for given run number of this dataset.
745  //
746  // \returns a pointer to the opened file; if the file is not available, we return nullptr.
747  //
748  // The user must cast the returned pointer to the correct class, which will
749  // depend on the data type and the dataset
750  //
751  // **SPECIAL CASE: MFM data with EBYEDAT frames**
752  //
753  // If the variable
754  //
755  //~~~~~~~~~~~~~~~~~~~~~~~~~
756  // [dataset].MFM.WithEbyedat: yes
757  //~~~~~~~~~~~~~~~~~~~~~~~~~
758  //
759  // is set, then we expect to find the necessary `ACTIONS_*` files in the dataset directory
760  // in subdirectory `ebyedat` (they should have the same names as the data files prefixed by
761  // `ACTIONS_[expname].CHC_PAR.`).
762  //
763  // If in addition the variable
764  //
765  //~~~~~~~~~~~~~~~~~~~~~~~~~
766  // [dataset].MFM.EbyedatActionsExpName: [expname]
767  //~~~~~~~~~~~~~~~~~~~~~~~~~
768  //
769  // is set, then we use the same `ACTIONS` file for all runs, with name `ACTIONS_[expname].CHC_PAR`
770 
771 
772  if (!strcmp(type, "raw") && !strcmp(GetDataSetEnv("MFM.WithEbyedat", ""), "yes")) {
773  TString ebydir = GetDataSetDir();
774  ebydir += "/ebyedat";
775  gEnv->SetValue("KVMFMDataFileReader.ActionsDirectory", ebydir);
776  if (strcmp(GetDataSetEnv("MFM.EbyedatActionsExpName", ""), ""))
777  gEnv->SetValue("KVMFMDataFileReader.ActionsExpName", GetDataSetEnv("MFM.EbyedatActionsExpName", ""));
778  TObject* f = GetRepository()->OpenDataSetRunFile(this, type, run, GetName());
779  // reset in case another dataset opens a raw MFM file without EBYEDAT data
780  gEnv->SetValue("KVMFMDataFileReader.ActionsDirectory", "");
781  gEnv->SetValue("KVMFMDataFileReader.ActionsExpName", "");
782  return f;
783  }
784  return GetRepository()->OpenDataSetRunFile(this, type, run, GetName());
785 }
786 
787 
788 
789 
793 
795  const run_index_t& run) const
796 {
797  //\return full path to file containing data of given datatype for given run/index of this dataset
798  // \note only works for available run files, if their is no file in the repository for this run, the returned path will be empty
799 
800  TString file("");
801  if (fRepository) file = GetRunfileName(type, run);
802  if (file == "")
803  return file.Data();
804  return fRepository->GetFullPathToOpenFile(this, type, file.Data());
805 }
806 
807 
808 
809 
813 
815 {
816  //\return name of file containing data of given datatype for given run/index of this dataset
817  //\note only works for available run files, if there is no file in the repository for this run, the returned path will be empty
818 
819  if (!HasDataType(type)) {
820  Error("GetRunfileName",
821  "No data of type \"%s\" available for dataset %s", (const char*)type,
822  GetName());
823  return 0;
824  }
825  //get name of file from available runs file
826  return GetAvailableRunsFile(type)->GetFileName(run);
827 }
828 
829 
830 
831 
837 
838 std::optional<TDatime> KVDataSet::GetRunfileDate(const KVString& type, const run_index_t& run)
839 {
840  //\return date of file containing data of given datatype for given run/index of this dataset
841  //
842  //\note only works for available runfiles, if there is no file in the repository for this run/index,
843  //an error will be printed and std::optional will not contain a value
844 
845  if (!HasDataType(type)) {
846  Error("GetRunfileDate",
847  "No data of type \"%s\" available for dataset %s", (const char*)type,
848  GetName());
849  return std::nullopt;
850  }
851  //get date of file from available runs file
852  TDatime date;
854  if (!GetAvailableRunsFile(type)->GetRunInfo(run, date, filename)) {
855  Error("GetRunfileDate",
856  "Runfile not found for run %d index %d (data type: %s)", run.run(), run.index(), (const char*)type);
857  return std::nullopt;
858  }
859  return date;
860 }
861 
862 
863 
864 
869 
871 {
872  //We check the availability of the run by looking in the available runs file associated
873  //with the given datatype.
874 
875  //check data type is available
876  if (!HasDataType(type)) {
877  Error("CheckRunfileAvailable",
878  "No data of type \"%s\" available for dataset %s", (const char*)type,
879  GetName());
880  return 0;
881  }
882  return GetAvailableRunsFile(type)->CheckAvailable(run);
883 }
884 
885 
886 
887 
902 
903 const Char_t* KVDataSet::GetBaseFileName(const Char_t* type, const run_index_t& run) const
904 {
905  //PRIVATE METHOD: Returns base name of data file containing data for the run of given datatype.
906  //The filename corresponds to one of the formats defined in $KVROOT/KVFiles/.kvrootrc
907  //by variables like:
908  //
909  //~~~
910  //[dataset].DataSet.RunFileName.[type]: run%R.dat
911  //~~~
912  //
913  //%R will be replaced with the run number
914  //
915  //IF the format contains '%D' it will be replaced with the current date and time
916  //
917  // Any index will be appended at the end: ".index"
918 
919  static TString tmp;
920  //get format string
921  TString fmt = GetDataSetEnv(Form("DataSet.RunFileName.%s", type));
922  TString run_num(Form("%d", run.run()));
923  KVDatime now;
924  TString date(now.AsSQLString());
925  tmp = fmt;
926  tmp.ReplaceAll("%R", run_num);
927  if (fmt.Contains("%D")) {
928  tmp.ReplaceAll("%D", date);
929  }
930  if (run.has_index()) tmp += Form(".%d", run.index());
931  return tmp.Data();
932 }
933 
934 
935 
936 
940 
942 {
943  //Update list of available runs for given data 'type'
944 
945  //check data type is available
946  if (!HasDataType(type)) {
947  Error("UpdateAvailableRuns",
948  "No data of type \"%s\" available for dataset %s", (const char*)type,
949  GetName());
950  }
951  KVAvailableRunsFile* a = GetAvailableRunsFile(type);
952  a->Update();
953 }
954 
955 
956 
957 
962 
964 {
965  // Create a new runfile for the dataset of given datatype.
966  // (only if this dataset is associated with a data repository)
967  // Once the file has been filled, use CommitRunfile to submit it to the repository.
968 
969  if (!fRepository) return nullptr;
970  TString tmp = GetBaseFileName(type, run);
971  //turn any spaces into "_"
972  tmp.ReplaceAll(" ", "_");
973  return fRepository->CreateNewFile(this, type, tmp.Data());
974 }
975 
976 
977 
978 
987 
988 void KVDataSet::DeleteRunfile(const KVString& type, const run_index_t& run, Bool_t confirm)
989 {
990  // Delete the file for the given run/index of data type "type" from the repository.
991  // By default, confirm=kTRUE, which means that the user will be asked to confirm
992  // that the file should be deleted. If confirm=kFALSE, no confirmation will be asked
993  // for and the file will be deleted straight away.
994  //
995  // WARNING: this really does DELETE files in the repository, they cannot be
996  // retrieved once they have been deleted.
997 
998  if (!fRepository) return;
999 
1000  //get name of file to delete
1001  TString filename = GetAvailableRunsFile(type)->GetFileName(run);
1002  if (filename == "") {
1003  Error("DeleteRunfile", "Run %s of type %s does not exist.",
1004  run.as_string().Data(), (const char*)type);
1005  return;
1006  }
1007  //delete file
1008  //prevent accidental deletion of certain types of runfiles
1009  KVString doNotDelete = GetDataSetEnv("DataSet.RunFile.DoNotDelete", "all");
1010  if (doNotDelete == "all" || doNotDelete.Contains(type)) {
1011  Error("DeleteRunFile", "%s files cannot be deleted", (const char*)type);
1012  return;
1013  }
1014  fRepository->DeleteFile(this, type, filename.Data(), confirm);
1015  //was file deleted ? if so, remove entry from available runs file
1016  if (!fRepository->CheckFileStatus(this, type, filename.Data()))
1017  GetAvailableRunsFile(type)->Remove(run);
1018 }
1019 
1020 
1021 
1022 
1034 
1035 void KVDataSet::DeleteRunfiles(const Char_t* type, const run_index_list& nl, Bool_t confirm)
1036 {
1037  // Delete files corresponding to a list of runs/index of data type "type" from the repository.
1038  //
1039  // By default, confirm=kTRUE, which means that the user will be asked to confirm
1040  // that each file should be deleted.
1041  //
1042  // If confirm=kFALSE, no confirmation will be asked for and the file will be deleted straight away.
1043  //
1044  // if "nl" is empty (default value) all runs of the dataset corresponding to the given type will be deleted
1045  //
1046  // WARNING: this really does DELETE files in the repository, they cannot be retrieved once they have been deleted.
1047 
1048  auto NL = nl;
1049  if (NL.IsEmpty()) NL = GetRunList(type);
1050  if (NL.IsEmpty()) return;
1051  for (auto& r : NL)
1052  DeleteRunfile(type, r, confirm);
1053 }
1054 
1055 
1056 
1057 
1065 
1066 run_index_list KVDataSet::GetRunList_DateSelection(const Char_t* type, TDatime* min, TDatime* max)
1067 {
1068  // Prints out and returns list of runs after date / time selection
1069  //
1070  // Runs generated between ]min;max[ are selected
1071  // - if min=NULL runs with date <max are selected
1072  // - if max=NULL runs with date >min are selected
1073  // - if max and min are NULL returns empty list
1074 
1075  if (!min && !max) return {};
1076 
1077  if (min) printf("date minimum %s\n", min->AsString());
1078  if (max) printf("date maximum %s\n", max->AsString());
1079 
1080  run_index_list numb;
1081 
1082  auto ll = GetListOfAvailableSystems(type);
1083  KVDBSystem* sys = 0;
1084  KVRunFile* run = 0;
1085  for (Int_t nl = 0; nl < ll->GetEntries(); nl += 1) {
1086  sys = (KVDBSystem*)ll->At(nl);
1087  auto lrun = GetListOfAvailableSystems(type, sys);
1088  auto oldList = numb;
1089  for (Int_t nr = 0; nr < lrun->GetEntries(); nr += 1) {
1090  run = (KVRunFile*)lrun->At(nr);
1091 
1092  if (min && max) {
1093  if (*min < run->GetFileWrittenDatime() && run->GetFileWrittenDatime() < *max) {
1094  numb.Add(run->GetRunIndex());
1095  }
1096  }
1097  else if (min) {
1098  if (*min < run->GetFileWrittenDatime()) {
1099  numb.Add(run->GetRunIndex());
1100  }
1101  }
1102  else if (max) {
1103  if (run->GetFileWrittenDatime() < *max) {
1104  numb.Add(run->GetRunIndex());
1105  }
1106  }
1107  }
1108  // print runs for system if any
1109  if (numb.GetEntries() > oldList.GetEntries())
1110  printf("%s : %s\n", sys->GetName(), (numb - oldList).AsString().Data());
1111  }
1112  return numb;
1113 
1114 }
1115 
1116 
1117 
1118 
1127 
1128 run_index_list KVDataSet::GetRunList_StageSelection(const Char_t* type, const Char_t* ref_type, KVDBSystem* system, Bool_t OnlyCol)
1129 {
1130  // Returns list of runs which are present for data type "base_type" but not for "other_type"
1131  //
1132  // if type is NULL or ="" returns empty KVNumberList
1133  //
1134  // If pointer to system is given, only runs for the system are considered.
1135  //
1136  // If OnlyCol=kTRUE (kFALSE default) only systems with KVDBSystem::IsCollision()=kTRUE are considered
1137 
1138  run_index_list manquant;
1139  auto ll = GetListOfAvailableSystems(ref_type);
1140  if (!ll || !ll->GetEntries()) {
1141  //numb.Clear();
1142  Info("GetRunList_StageSelection", "No data available of type \"%s\"", ref_type);
1143  return manquant;
1144  }
1145  if (system && !ll->FindObject(system)) {
1146  Info("GetRunList_StageSelection", "No data available of type \"%s\" for system %s", ref_type, system->GetName());
1147  return manquant;
1148  }
1149 
1150  Info("GetRunList_StageSelection", "Liste des runs presents dans \"%s\" mais absent dans \"%s\"", ref_type, type);
1151 
1152  KVDBSystem* sys = 0;
1153 
1154  for (Int_t nl = 0; nl < ll->GetEntries(); nl += 1) {
1155 
1156  sys = (KVDBSystem*)ll->At(nl);
1157  if (system && sys != system) continue;
1158  if (OnlyCol && !sys->IsCollision()) continue;
1159  auto nsys = GetRunList(type, sys);
1160  auto nsys_ref = GetRunList(ref_type, sys);
1161  Int_t nref = nsys_ref.GetNValues();
1162 
1163  nsys_ref.Remove(nsys);
1164 
1165  Info("GetRunList_StageSelection", "\nKVDBSystem : %s --> %d runs manquants sur %d : %s",
1166  sys->GetName(),
1167  nsys_ref.GetNValues(),
1168  nref,
1169  nsys_ref.AsString().Data()
1170  );
1171  manquant.Add(nsys_ref);
1172  }
1173  return manquant;
1174 }
1175 
1176 
1177 
1181 
1182 run_index_list KVDataSet::GetRunList_VersionSelection(const Char_t* type, const Char_t* version, KVDBSystem* sys)
1183 {
1184  // Returns list of runs of given type that were created with the given version of KaliVeda.
1185  // If system!="" then only runs for the given system are considered
1186 
1187  run_index_list runs;
1188  if (sys) {
1189  unique_ptr<TList> lrun(GetListOfAvailableSystems(type, sys));
1190  TIter next(lrun.get());
1191  KVRunFile* run;
1192  while ((run = (KVRunFile*)next())) {
1193  if (!strcmp(run->GetVersion(), version)) runs.Add(run->GetRunIndex());
1194  }
1195  return runs;
1196  }
1197  unique_ptr<TList> ll(GetListOfAvailableSystems(type));
1198  if (!ll.get() || !ll->GetEntries()) {
1199  //numb.Clear();
1200  Info("GetRunList_VersionSelection", "No data available of type \"%s\"", type);
1201  return runs;
1202  }
1203  Int_t nsys = ll->GetEntries();
1204  for (Int_t nl = 0; nl < nsys; nl += 1) {
1205  sys = (KVDBSystem*)ll->At(nl);
1206  unique_ptr<TList> lrun(GetListOfAvailableSystems(type, sys));
1207  TIter next(lrun.get());
1208  KVRunFile* run;
1209  while ((run = (KVRunFile*)next())) {
1210  if (!strcmp(run->GetVersion(), version)) runs.Add(run->GetRunIndex());
1211  }
1212  }
1213  return runs;
1214 }
1215 
1216 
1217 
1222 
1223 void KVDataSet::CommitRunfile(const KVString& type, const run_index_t& run, TFile* file)
1224 {
1225  // Commit a runfile previously created with NewRunfile() to the repository.
1226  // Any previous version of the runfile will be deleted.
1227  // The available runs list for this data 'type' is updated.
1228 
1229  if (!fRepository) return;
1230 
1231  //keep name of file for updating available runs list
1232  TString newfile = gSystem->BaseName(file->GetName());
1233 
1234  fRepository->CommitFile(file, type, this);
1235  //update list of available datatypes of dataset,
1236  //in case this addition has created a new subdirectory
1237  CheckAvailable();
1238  //check if previous version of file exists
1239  //get name of file from available runs file
1240  //note that when the file is the first of a new subdirectory, GetAvailableRunsFile->GetFileName
1241  //will cause the available runs file to be created, and it will contain one entry:
1242  //the new file!
1243  TString oldfile = GetAvailableRunsFile(type)->GetFileName(run);
1244  if (oldfile != "" && oldfile != newfile) {
1245  //delete previous version - no confirmation
1246  fRepository->DeleteFile(this, type, oldfile.Data(),
1247  kFALSE);
1248  //was file deleted ? if so, remove entry from available runs file
1249  if (!fRepository->CheckFileStatus(this, type, oldfile.Data()))
1250  GetAvailableRunsFile(type)->Remove(run);
1251  }
1252  if (oldfile != newfile) {
1253  //add entry for new run in available runs file
1254  GetAvailableRunsFile(type)->Add(run, newfile.Data());
1255  }
1256 }
1257 
1258 
1259 
1260 
1266 
1268 {
1269  //if fUserGroups has been set with SetUserGroups(), we check that the current user's name
1270  //(gSystem->GetUserInfo()->fUser) appears in at least one of the groups in the list.
1271  //Returns kFALSE if user's name is not found in any of the groups.
1272  //if fUserGroups="" (default), we return kTRUE for all users.
1273 
1274  if (fUserGroups == "")
1275  return kTRUE; /* no groups set, all users have access */
1276 
1277  //split into array of group names
1278  unique_ptr<TObjArray> toks(fUserGroups.Tokenize(' '));
1279  TObjString* group_name;
1280  TIter next_name(toks.get());
1281  while ((group_name = (TObjString*) next_name())) {
1282  //for each group_name, we check if the user's name appears in the group
1283  if (!fRepository || (fRepository && fRepository->GetDataSetManager()->
1284  CheckUser(group_name->String().Data()))
1285  ) {
1286  return kTRUE;
1287  }
1288  }
1289  return kFALSE;
1290 }
1291 
1292 
1293 
1294 
1297 
1299 {
1300  //Set pointer to data repository in which dataset is stored
1301  fRepository = dr;
1302 }
1303 
1304 
1305 
1306 
1309 
1311 {
1312  //Get pointer to data repository in which dataset is stored
1313  return fRepository;
1314 }
1315 
1316 
1317 
1318 
1323 
1325 {
1326  //Check all runs for a given datatype and make sure that only one version
1327  //exists for each runfile. If not, we print a report on the runfiles which occur
1328  //multiple times, with the associated date and file name.
1329 
1330  auto doubles = GetAvailableRunsFile(data_type)->CheckMultiRunfiles();
1331  if (doubles.empty()) {
1332  cout << "OK. No runs appear more than once." << endl;
1333  }
1334  else {
1335  cout << "Runs which appear more than once: " << endl << endl;
1336  //print dates and filenames for each run
1337 
1338  for (auto& rr : doubles) {
1339  KVList filenames, dates;
1340 
1341  //get infos for current run
1342  GetAvailableRunsFile(data_type)->GetRunInfos(rr, &dates, &filenames);
1343 
1344  cout << "Run " << rr << " : " << dates.GetEntries() << " files >>>>>>" << endl;
1345  for (int i = 0; i < dates.GetEntries(); i++) {
1346 
1347  cout << "\t" << ((TObjString*) filenames.At(i))->String().
1348  Data() << "\t" << ((TObjString*) dates.At(i))->String().
1349  Data() << endl;
1350 
1351  }
1352  }
1353  }
1354 }
1355 
1356 
1357 
1358 
1367 
1368 void KVDataSet::CleanMultiRunfiles(const Char_t* data_type, Bool_t confirm)
1369 {
1370  // Check all runs for a given datatype and make sure that only one version
1371  // exists for each run. If not, we print a report on the runfiles which occur
1372  // multiple times, with the associated date and file name, and then we
1373  // destroy all but the most recent version of the file in the repository, and
1374  // update the runlist accordingly.
1375  // By default, we ask for confirmation before deleting each file.
1376  // Call with confirm=kFALSE to delete WITHOUT CONFIRMATION (DANGER!! WARNING!!!)
1377 
1378  if (!fRepository) return;
1379 
1380  KVAvailableRunsFile* ARF = GetAvailableRunsFile(data_type);
1381  auto doubles = ARF->CheckMultiRunfiles();
1382  if (doubles.empty()) {
1383  cout << "OK. No runs appear more than once." << endl;
1384  }
1385  else {
1386  cout << "Runs which appear more than once: " << endl << endl;
1387  //print dates and filenames for each run
1388 
1389  KVList filenames, dates;
1390  for (auto& rr : doubles) {
1391  //get infos for current run
1392  ARF->GetRunInfos(rr, &dates, &filenames);
1393  TDatime most_recent("1998-12-25 00:00:00");
1394  Int_t i_most_recent = 0;
1395  cout << "Run " << rr << " : " << dates.GetEntries() << " files >>>>>>" << endl;
1396  for (int i = 0; i < dates.GetEntries(); i++) {
1397  //check if run is most recent
1398  TDatime rundate(((TObjString*) dates.At(i))->String().Data());
1399  if (rundate > most_recent) {
1400  most_recent = rundate;
1401  i_most_recent = i;
1402  }
1403  }
1404  //Now, we loop over the list again, this time we destroy all but the most recent
1405  //version of the runfile
1406  for (int i = 0; i < dates.GetEntries(); i++) {
1407  if (i == i_most_recent) {
1408  cout << "KEEP : ";
1409  }
1410  else {
1411  cout << "DELETE : ";
1412  }
1413  cout << "\t" << ((TObjString*) filenames.At(i))->String().
1414  Data() << "\t" << ((TObjString*) dates.At(i))->String().
1415  Data() << endl;
1416  if (i != i_most_recent) {
1417  //delete file from repository forever and ever
1418  fRepository->DeleteFile(this, data_type,
1419  ((TObjString*) filenames.At(i))->
1420  String().Data(), confirm);
1421  //remove file entry from available runlist
1422  ARF->Remove(rr,
1423  ((TObjString*) filenames.At(i))->String());
1424  }
1425  }
1426  }
1427  }
1428 }
1429 
1430 
1431 
1432 
1438 
1440  KVDataRepository* other_repos)
1441 {
1442  //Use this method to check whether the file of type "data_type" for run number "run"
1443  //in the data repository "other_repos" is more recent than the file contained in the data
1444  //repository corresponding to this dataset.
1445  //Returns kFALSE if file in other repository is more recent.
1446 
1447  if (!other_repos)
1448  return kTRUE;
1449  //get dataset with same name as this one from dataset manager of other repository
1450  KVDataSet* ds = other_repos->GetDataSetManager()->GetDataSet(GetName());
1451  if (!ds) {
1452  Error("CheckRunfileUpToDate",
1453  "Dataset \"%s\" not found in repository \"%s\"", GetName(),
1454  other_repos->GetName());
1455  return kFALSE;
1456  }
1457  //compare dates of the two runfiles
1458  if (GetRunfileDate(data_type, run) < ds->GetRunfileDate(data_type, run))
1459  return kFALSE;
1460  return kTRUE;
1461 }
1462 
1463 
1464 
1465 
1472 
1474  const KVString& other_repos)
1475 {
1476  //Use this method to check whether the file of type "data_type" for run number "run"
1477  //in the data repository "other_repos" is more recent than the file contained in the data
1478  //repository corresponding to this dataset.
1479  //Returns kTRUE if no repository with name "other_repos" exists.
1480  //Returns kFALSE if file in other repository is more recent.
1481 
1482  KVDataRepository* _or =
1483  gDataRepositoryManager->GetRepository(other_repos);
1484  if (_or)
1485  return CheckRunfileUpToDate(data_type, run, _or);
1486  Error("CheckRunfileUpToDate",
1487  "No data repository known with this name : %s", (const char*) other_repos);
1488  return kTRUE;
1489 }
1490 
1491 
1492 
1493 
1497 
1498 void KVDataSet::CheckUpToDate(const Char_t* data_type,
1499  const Char_t* other_repos)
1500 {
1501  //Check whether all files of type "data_type" for run number "run" in the data repository
1502  //are up to date (i.e. at least as recent) as compared to the files in data repository "other_repos".
1503 
1504  if (!fRepository) return;
1505 
1506  KVDataRepository* _or =
1507  gDataRepositoryManager->GetRepository(other_repos);
1508  if (!_or) {
1509  Error("CheckUpToDate",
1510  "No data repository known with this name : %s", other_repos);
1511  return;
1512  }
1513  auto runlist = GetAvailableRunsFile(data_type)->GetRunList();
1514  Int_t need_update = 0;
1515  for (auto& rr : runlist) {
1516  //check run
1517  if (!CheckRunfileUpToDate(data_type, rr, _or)) {
1518  cout << " *** run " << rr << " needs update ***" <<
1519  endl;
1520  cout << "\t\tREPOSITORY: " << fRepository->
1521  GetName() << "\tDATE: " << GetRunfileDate(data_type, rr).value().AsString() << endl;
1522  cout << "\t\tREPOSITORY: " << other_repos << "\tDATE: " << _or->
1523  GetDataSetManager()->GetDataSet(GetName())->
1524  GetRunfileDate(data_type, rr).value().AsString() << endl;
1525  need_update++;
1526  }
1527  }
1528  if (!need_update) {
1529  cout << " *** All runfiles are up to date for data type " <<
1530  data_type << endl;
1531  }
1532 }
1533 
1534 
1535 
1536 
1540 
1541 run_index_list KVDataSet::GetUpdatableRuns(const Char_t* data_type,
1542  const Char_t* other_repos)
1543 {
1544  //Returns list of all runs of type "data_type" which may be updated
1545  //from the repository named "other_repos". See CheckUpToDate().
1546 
1547  run_index_list updates;
1548  if (!fRepository) return updates;
1549 
1550  KVDataRepository* _or =
1551  gDataRepositoryManager->GetRepository(other_repos);
1552  if (!_or) {
1553  Error("CheckUpToDate",
1554  "No data repository known with this name : %s", other_repos);
1555  return updates;
1556  }
1557  auto runlist = GetAvailableRunsFile(data_type)->GetRunList();
1558  for (auto& rr : runlist) {
1559  //check run
1560  if (!CheckRunfileUpToDate(data_type, rr, _or)) {
1561  //run is out of date
1562  updates.Add(rr);
1563  }
1564  }
1565  return updates;
1566 }
1567 
1568 
1569 
1570 
1575 
1576 run_index_list KVDataSet::GetRunList(const Char_t* data_type,
1577  const KVDBSystem* system) const
1578 {
1579  //Returns list of all files available for given "data_type"
1580  //
1581  //If a pointer to a reaction system is given, only files for the given system will be included in the list.
1582 
1583  if (!fRepository || !HasDataType(data_type)) {
1584  Error("GetRunList",
1585  "No data of type %s available. Runlist will be empty.",
1586  data_type);
1587  return {};
1588  }
1589 
1590  return GetAvailableRunsFile(data_type)->GetRunList(system);
1591 }
1592 
1593 
1594 
1595 
1626 
1628 {
1629  //This method returns a pointer to the available analysis task whose description (title) contains
1630  //all of the whitespace-separated keywords (which may be regular expressions)
1631  //given in the string "keywords". The comparison is case-insensitive.
1632  //
1633  //WARNING: this method can only be used to access analysis tasks that are
1634  //available for this dataset, i.e. for which the corresponding prerequisite data type
1635  //is available in the repository.
1636  //For unavailable data/tasks, use GetAnalysisTaskAny(const Char_t*).
1637  //
1638  //EXAMPLES
1639  //Let us suppose that the current dataset has the following list of tasks:
1640  //~~~
1641  // root [2] gDataSet->Print("tasks")
1642  // 1. Event reconstruction from raw data (raw->recon)
1643  // 2. Analysis of raw data
1644  // 3. Identification of reconstructed events (recon->ident)
1645  // 4. Analysis of reconstructed events (recon)
1646  // 5. Analysis of partially identified & calibrated reconstructed events (ident)
1647  // 6. Analysis of fully calibrated physical data (root)
1648  //~~~
1649  //Then the following will occur:
1650  //~~~
1651  // root [14] gDataSet->GetAnalysisTask("raw->recon")->Print()
1652  // KVDataAnalysisTask : Event reconstruction from raw data (raw->recon)
1653  //
1654  // root [10] gDataSet->GetAnalysisTask("analysis root")->Print()
1655  // KVDataAnalysisTask : Analysis of fully calibrated physical data (root)
1656  //~~~
1657 
1658  //case-insensitive search for matches in list based on 'title' attribute
1659  return (KVDataAnalysisTask*)fTasks.FindObjectAny("title", keywords, kTRUE, kFALSE);
1660 }
1661 
1662 
1663 
1664 
1666 
1668 {
1669  if (!dat->WithUserClass()) {
1670  Error("MakeAnalysisClass",
1671  "No user class required for analysis task \"%s\"", dat->GetTitle());
1672  return false;
1673  }
1674 
1675  //all analysis base classes must define a static Make(const Char_t * classname)
1676  //which generates the skeleton class files.
1677 
1678  TClass* cl = nullptr;
1679  //has the user base class for the task been compiled and loaded ?
1681  else
1682  return false;
1683 
1684  //set up call to static Make method
1685  unique_ptr<TMethodCall> methcall(new TMethodCall(cl, "Make", Form("\"%s\"", classname)));
1686 
1687  if (!methcall->IsValid()) {
1688  Error("MakeAnalysisClass", "static Make(const Char_t*) method for class %s is not valid",
1689  cl->GetName());
1690  return false;
1691  }
1692 
1693  //generate skeleton class
1694  methcall->Execute();
1695 
1696  return true;
1697 }
1698 
1699 
1700 
1713 
1714 bool KVDataSet::MakeAnalysisClass(const Char_t* task, const Char_t* classname)
1715 {
1716  //Create a skeleton analysis class to be used for analysis of the data belonging to this dataset.
1717  //
1718  // task = keywords contained in title of analysis task (see GetAnalysisTask(const Char_t*))
1719  // (you do not need to include 'analysis', it is added automatically)
1720  // classname = name of new analysis class
1721  //
1722  //Example:
1723  // MakeAnalysisClass("raw", "MyRawDataAnalysis")
1724  // --> make skeleton raw data analysis class in files MyRawDataAnalysis.cpp & MyRawDataAnalysis.h
1725  // MakeAnalysisClass("fully calibrated", "MyDataAnalysis")
1726  // --> make skeleton data analysis class in files MyDataAnalysis.cpp & MyDataAnalysis.h
1727 
1728  KVString _task = task;
1729  if (!_task.Contains("nalysis")) _task += " analysis";
1730  //We want to be able to write analysis classes even when we don't have any data
1731  //to analyse. Therefore we use GetAnalysisTaskAny.
1732  auto dat = GetAnalysisTaskAny(_task.Data());
1733  if (!dat) {
1734  Error("MakeAnalysisClass",
1735  "called for unknown or unavailable analysis task : %s", _task.Data());
1736  return false;
1737  }
1738  return make_analysis_class(dat, classname);
1739 }
1740 
1741 
1742 
1749 
1750 bool KVDataSet::MakeAnalysisClass(int task, const Char_t* classname)
1751 {
1752  //Create a skeleton analysis class to be used for analysis of the data belonging to this dataset.
1753  //
1754  // task = index of analysis task as shown in KVDataSet::Print("tasks")
1755  // classname = name of new analysis class
1756  //
1757 
1758  auto dat = GetAnalysisTask(task);
1759  if (!dat) {
1760  Error("MakeAnalysisClass",
1761  "called for unknown or unavailable analysis task index : %d", task);
1762  return false;
1763  }
1764  return make_analysis_class(dat, classname);
1765 }
1766 
1767 
1768 
1769 
1773 
1774 Bool_t KVDataSet::OpenDataSetFile(const Char_t* filename, ifstream& file)
1775 {
1776  // Look for (and open for reading, if found) the named file in the directory which
1777  // contains the files for this dataset (given by GetDataSetDir())
1778 
1779  return OpenDataSetFile(GetName(), filename, file);
1780 }
1781 
1782 
1783 
1787 
1788 Bool_t KVDataSet::OpenDataSetFile(const TString& dataset, const Char_t* filename, ifstream& file)
1789 {
1790  // Static method to look for (and open for reading, if found) the named file in the directory which
1791  // contains the files for the dataset
1792 
1793  TString datasetdir = KVBase::GetDataSetEnv(dataset, "DataSet.Directory", dataset);
1794  if (gSystem->IsAbsoluteFileName(datasetdir)) {
1795  // dataset directory is outside of standard KV installation directories
1796  // use absolute path to search for file
1797  TString abspath;
1798  abspath.Form("%s/%s", datasetdir.Data(), filename);
1799  return SearchAndOpenKVFile(abspath, file);
1800  }
1801  // dataset directory is a subdirectory of GetDATADIRFilePath()
1802  return SearchAndOpenKVFile(filename, file, datasetdir);
1803 }
1804 
1805 
1806 
1810 
1812 {
1813  // Find a file in the dataset directory (given by GetDataSetDir())
1814  // Returns full path to file if found, empty string if not
1815 
1816  return GetFullPathToDataSetFile(GetName(), filename);
1817 }
1818 
1819 
1820 
1824 
1826 {
1827  // Static method to find a file in the dataset directory (given by GetDataSetDir())
1828  // Returns full path to file if found, empty string if not
1829 
1830  TString fullpath;
1831  TString datasetdir = KVBase::GetDataSetEnv(dataset, "DataSet.Directory", dataset);
1832  if (!SearchKVFile(filename, fullpath, datasetdir)) {
1833  ::Warning("KVDataSet::GetFullPathToDataSetFile", "File %s not found in dataset subdirectory %s", filename, datasetdir.Data());
1834  fullpath = "";
1835  }
1836  return fullpath;
1837 }
1838 
1839 
1840 
1844 
1845 Bool_t KVDataSet::FindDataSetFile(const TString& dataset, const Char_t* filename)
1846 {
1847  // Static method to find a file in the dataset directory (given by GetDataSetDir())
1848  // Returns kTRUE if found, kFALSE if not
1849 
1850  TString fullpath;
1851  TString datasetdir = KVBase::GetDataSetEnv(dataset, "DataSet.Directory", dataset);
1852  return SearchKVFile(filename, fullpath, datasetdir);
1853 }
1854 
1855 
1856 
1860 
1862 {
1863  // Find a file in the dataset directory (given by GetDataSetDir())
1864  // Returns kTRUE if found, kFALSE if not
1865 
1866  return FindDataSetFile(GetName(), filename);
1867 }
1868 
1869 
1870 
1871 
1880 
1882 {
1883  //This method returns the analysis task whose description (title) contains
1884  //all of the whitespace-separated keywords (which may be regular expressions)
1885  //given in the string "keywords". The comparison is case-insensitive.
1886  //The analysis task does not need to be "available", i.e. the associated prerequisite
1887  //data type does not have to be present in the repository (see GetAnalysisTask).
1888  //
1889  // If no task is found, returns nullptr
1890 
1891  KVDataAnalysisTask* tsk = (KVDataAnalysisTask*)fAllTasks.FindObjectAny("title", keywords, kTRUE, kFALSE);
1892  if (!tsk) {
1893  Error("GetAnalysisTaskAny", "No task found with the following keywords in its title : %s", keywords);
1894  }
1895  return tsk;
1896 }
1897 
1898 
1899 
1900 
1906 
1908 {
1909  // Returns kTRUE if database needs to be regenerated from source files,
1910  // i.e. if source files are more recent than DataBase.root
1911  // In case no directory exists for dataset (dataset added 'on the fly')
1912  // we create the directory and fill it with dummy files (Makefile, Runlist.csv, Systems.dat)
1913 
1914  TString pwd = gSystem->pwd();
1915 
1916  TString path = "";
1917  if (!SearchKVFile(GetDataSetDir(), path)) {
1918  // dataset directory doesn't exist - create it
1919  Info("DataBaseNeedsUpdate", "%s: Creating new dataset directory %s",
1920  GetName(), GetDataSetDir());
1921  if (gSystem->mkdir(GetDataSetDir())) {
1922  // problem creating directory
1923  Error("DataBaseNeedsUpdate",
1924  "%s: Dataset directory %s does not exist and cannot be created ?",
1925  GetName(), GetDataSetDir());
1926  return kFALSE;
1927  }
1928  // create dummy files
1929  SearchKVFile(GetDataSetDir(), path); // get full path
1930  path += "/";
1931  TString filename = path + "Makefile";
1932  ofstream of1(filename.Data());
1933  of1 << "$(KV_WORK_DIR)/db/" << GetName() << "/DataBase.root : Runlist.csv Systems.dat" << endl;
1934  of1 << "\t@echo Database needs update" << endl;
1935  of1.close();
1936  filename = path + "Runlist.csv";
1937  ofstream of2(filename.Data());
1938  of2 << "# Automatically generated dummy Runlist.csv file" << endl;
1939  of2.close();
1940  filename = path + "Systems.dat";
1941  ofstream of3(filename.Data());
1942  of3 << "# Automatically generated dummy Systems.dat file" << endl;
1943  of3.close();
1944  }
1945  gSystem->cd(GetDataSetDir());
1946  TString cmd = "make -q";
1947  Int_t ret = gSystem->Exec(cmd.Data());
1948  gSystem->cd(pwd.Data());
1949  return (ret != 0);
1950 }
1951 
1952 
1953 
1970 
1972 {
1973  // Returns name of output repository for given task.
1974  // By default it is the name of the repository associated with this dataset,
1975  // but can be changed by the following environment variables:
1976  //
1977  // [repository].DefaultOutputRepository: [other repository]
1978  // - this means that all tasks carried out on data in [repository]
1979  // will have their output files placed in [other repository]
1980  //
1981  // [taskname].DataAnalysisTask.OutputRepository: [other repository]
1982  // - this means that for [taskname], any output files will
1983  // be placed in [other repository]
1984  //
1985  // [dataset].[taskname].DataAnalysisTask.OutputRepository: [other repository]
1986  // - this means that for given [dataset] & [taskname],
1987  // any output files will be placed in [other repository]
1988 
1989  if (gEnv->Defined(Form("%s.DataRepository.DefaultOutputRepository", GetRepository()->GetName())))
1990  return TString(gEnv->GetValue(Form("%s.DataRepository.DefaultOutputRepository", GetRepository()->GetName()), ""));
1991  TString orep = GetDataSetEnv(Form("%s.DataAnalysisTask.OutputRepository", taskname), GetRepository()->GetName());
1992  return orep;
1993 }
1994 
1995 
1996 
1997 
2000 
2001 void KVDataSet::CopyRunfilesFromRepository(const Char_t* type, const run_index_list& runs, const Char_t* destdir)
2002 {
2003  // Copies the runfiles of given "type" into the local directory "destdir".
2004 
2005  KVDataRepository* repo = GetRepository();
2006  for (auto& run : runs) {
2007  TString filename = GetRunfileName(type, run);
2008  TString destpath;
2009  AssignAndDelete(destpath, gSystem->ConcatFileName(destdir, filename));
2010  repo->CopyFileFromRepository(this, type, filename, destpath);
2011  }
2012 }
2013 
2014 
2015 
2016 
2020 
2021 void KVDataSet::CopyRunfilesToRepository(const Char_t* type, const run_index_list& runs, const Char_t* destrepo)
2022 {
2023  // Copies the runfiles of given "type" from the data repository associated
2024  // with this dataset into the local repository "destrepo".
2025 
2026  KVDataRepository* repo = GetRepository();
2027  KVDataRepository* dest_repo = gDataRepositoryManager->GetRepository(destrepo);
2028 
2029  if (!dest_repo) {
2030  Error("CopyRunfilesToRepository", "Unknown destination repository : %s", destrepo);
2031  gDataRepositoryManager->Print();
2032  return;
2033  }
2034 
2035  KVDataSet* dest_ds = dest_repo->GetDataSetManager()->GetDataSet(GetName());
2036  dest_repo->CreateAllNeededSubdirectories(dest_ds, type);
2037  for (auto& run : runs) {
2038  TString filename = GetRunfileName(type, run);
2039  TString destpath = dest_repo->GetFullPathToTransferFile(dest_ds, type, filename);
2040  repo->CopyFileFromRepository(this, type, filename, destpath);
2041  }
2042 }
2043 
2044 
int Int_t
ROOT::R::TRInterface & r
#define f(i)
bool Bool_t
char Char_t
constexpr Bool_t kFALSE
constexpr Bool_t kTRUE
const char Option_t
#define gDirectory
R__EXTERN TEnv * gEnv
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
char * Form(const char *fmt,...)
void AssignAndDelete(TString &target, char *tobedeleted)
R__EXTERN TSystem * gSystem
Handles lists of available runs for different datasets and types of data.
virtual void Remove(const run_index_t &run, const KVString &filename="")
Remove from the db ALL entries corresponding to the given run/index (and filename if given)
std::forward_list< run_index_t > CheckMultiRunfiles()
Returns a list with all run/indexes for which more than one file is in the available runs db.
void GetRunInfos(const run_index_t &run, KVList *dates, KVList *names)
virtual void Add(const run_index_t &run, const KVString &filename)
static ValType GetDataSetEnv(const KVString &dataset, const KVString &type, const ValType &defval)
Definition: KVBase.h:304
Database class used to store information on different colliding systems studied during an experiment....
Definition: KVDBSystem.h:52
Bool_t IsCollision() const
retourne kTRUE, si le systeme est une collision ie projectile+cible
Definition: KVDBSystem.cpp:105
Define and manage data analysis tasks.
virtual void SetDataAnalyser(const Char_t *d)
virtual Bool_t WithUserClass() const
virtual void SetPrereq(const Char_t *p)
virtual void SetStatusUpdateInterval(Long64_t n)
virtual const Char_t * GetUserBaseClass() const
virtual Bool_t CheckUserBaseClassIsLoaded() const
void SetExtraAClicIncludes(const KVString &list)
virtual const Char_t * GetPrereq() const
virtual void SetUserBaseClass(const Char_t *d)
KVDataRepository * GetRepository(const Char_t *name) const
void Print(Option_t *opt="") const
Base class for managing repositories of experimental data.
virtual KVDataSetManager * GetDataSetManager() const
Return pointer to data set manager for this repository.
virtual void CopyFileFromRepository(const KVDataSet *dataset, const Char_t *datatype, const Char_t *filename, const Char_t *destination)
void CreateAllNeededSubdirectories(const KVDataSet *DataSet, const Char_t *DataType)
virtual const Char_t * GetFullPathToTransferFile(const KVDataSet *dataset, const Char_t *datatype, const Char_t *runfile)
KVDataSet * GetDataSet(Int_t) const
Return pointer to DataSet using index in list of all datasets, index>=0.
Manage an experimental dataset corresponding to a given experiment or campaign.
Definition: KVDataSet.h:130
std::optional< TDatime > GetRunfileDate(const KVString &type, const run_index_t &run)
Definition: KVDataSet.cpp:838
TString GetFullPathToRunfile(const KVString &type, const run_index_t &run) const
Definition: KVDataSet.cpp:794
TString GetFullPathToDB() const
Definition: KVDataSet.cpp:153
void ls(Option_t *opt="") const override
Print dataset information.
Definition: KVDataSet.cpp:377
TString GetOutputRepository(const Char_t *taskname) const
Definition: KVDataSet.cpp:1971
virtual void OpenDBFile(const Char_t *full_path_to_dbfile) const
Open the database from a file on disk.
Definition: KVDataSet.cpp:127
run_index_list GetRunList_VersionSelection(const Char_t *type, const Char_t *version, KVDBSystem *sys=0)
Definition: KVDataSet.cpp:1182
TString GetDBName() const
Definition: KVDataSet.cpp:112
void CheckMultiRunfiles(const Char_t *data_type)
Definition: KVDataSet.cpp:1324
Bool_t CheckRunfileAvailable(const KVString &type, const run_index_t &run)
Definition: KVDataSet.cpp:870
virtual const Char_t * GetBaseFileName(const Char_t *type, const run_index_t &run) const
Definition: KVDataSet.cpp:903
void CheckUpToDate(const Char_t *data_type, const Char_t *other_repos)
Definition: KVDataSet.cpp:1498
run_index_list GetRunList(const Char_t *data_type, const KVDBSystem *sys=0) const
Definition: KVDataSet.cpp:1576
void CopyRunfilesFromRepository(const Char_t *type, const run_index_list &runs, const Char_t *destdir)
Copies the runfiles of given "type" into the local directory "destdir".
Definition: KVDataSet.cpp:2001
KVDataRepository * GetRepository() const
Get pointer to data repository in which dataset is stored.
Definition: KVDataSet.cpp:1310
const Char_t * GetDataSetDir() const
Definition: KVDataSet.cpp:670
KVAvailableRunsFile * GetAvailableRunsFile(const Char_t *type) const
Definition: KVDataSet.cpp:51
void CommitRunfile(const KVString &type, const run_index_t &run, TFile *file)
Definition: KVDataSet.cpp:1223
virtual void AddAvailableDataType(const Char_t *)
Definition: KVDataSet.cpp:490
bool make_analysis_class(const KVDataAnalysisTask *, const Char_t *classname)
Definition: KVDataSet.cpp:1667
KVDataAnalysisTask * GetAnalysisTask(Int_t) const
Definition: KVDataSet.cpp:556
virtual void SaveDataBase() const
Definition: KVDataSet.cpp:203
virtual std::unique_ptr< TList > GetListOfAvailableSystems(const Char_t *datatype, KVDBSystem *systol=0)
Definition: KVDataSet.cpp:573
virtual KVDataAnalysisTask * GetAnalysisTaskAny(const Char_t *keywords) const
Definition: KVDataSet.cpp:1881
run_index_list GetRunList_DateSelection(const Char_t *type, TDatime *min=0, TDatime *max=0)
Definition: KVDataSet.cpp:1066
virtual void SetAnalysisTasks(const KVSeqCollection *)
Definition: KVDataSet.cpp:507
TFile * NewRunfile(const KVString &type, const run_index_t &run)
Definition: KVDataSet.cpp:963
Bool_t CheckRunfileUpToDate(const KVString &data_type, const run_index_t &run, KVDataRepository *other_repos)
Definition: KVDataSet.cpp:1439
virtual void CheckAvailable()
Definition: KVDataSet.cpp:443
void DeleteRunfile(const KVString &type, const run_index_t &run, Bool_t confirm=kTRUE)
Definition: KVDataSet.cpp:988
virtual Int_t GetNtasks() const
Definition: KVDataSet.cpp:541
run_index_list GetRunList_StageSelection(const Char_t *other_type, const Char_t *base_type, KVDBSystem *sys=0, Bool_t OnlyCol=kFALSE)
Definition: KVDataSet.cpp:1128
TString GetFullPathToDataSetFile(const Char_t *filename)
Definition: KVDataSet.cpp:1811
void DeleteRunfiles(const Char_t *type, const run_index_list &lrun={}, Bool_t confirm=kTRUE)
Definition: KVDataSet.cpp:1035
TObject * open_runfile(const KVString &type, const run_index_t &run)
Definition: KVDataSet.cpp:742
void CopyRunfilesToRepository(const Char_t *type, const run_index_list &runs, const Char_t *destrepo)
Definition: KVDataSet.cpp:2021
KVString GetRunfileName(const KVString &type, const run_index_t &run) const
Definition: KVDataSet.cpp:814
KVExpDB * GetDataBase(Option_t *opt="") const
Definition: KVDataSet.cpp:285
virtual Bool_t DataBaseNeedsUpdate() const
Definition: KVDataSet.cpp:1907
Bool_t OpenDataSetFile(const Char_t *filename, std::ifstream &file)
void SetDataSetSpecificTaskParameters(KVDataAnalysisTask *) const
Check configuration variables to see if the task parameters have been "tweaked" for the dataset.
Definition: KVDataSet.cpp:178
void cd() const
Definition: KVDataSet.cpp:692
virtual void WriteDBFile(const Char_t *full_path_to_dbfile) const
Write the database to disk.
Definition: KVDataSet.cpp:255
void Print(Option_t *opt="") const override
Definition: KVDataSet.cpp:400
void SetName(const char *name) override
Definition: KVDataSet.cpp:632
virtual Bool_t CheckUserCanAccess()
Definition: KVDataSet.cpp:1267
void UpdateAvailableRuns(const KVString &type)
Definition: KVDataSet.cpp:941
run_index_list GetUpdatableRuns(const Char_t *data_type, const Char_t *other_repos)
Definition: KVDataSet.cpp:1541
void SetRepository(KVDataRepository *)
Set pointer to data repository in which dataset is stored.
Definition: KVDataSet.cpp:1298
bool MakeAnalysisClass(const Char_t *task, const Char_t *classname)
Definition: KVDataSet.cpp:1714
KVDataSet()
Default constructor.
Definition: KVDataSet.cpp:35
void CleanMultiRunfiles(const Char_t *data_type, Bool_t confirm=kTRUE)
Definition: KVDataSet.cpp:1368
virtual void OpenDataBase(Option_t *opt="") const
Definition: KVDataSet.cpp:319
static Bool_t FindDataSetFile(const TString &dataset, const Char_t *filename)
Definition: KVDataSet.cpp:1845
TString GetDBFileName() const
Definition: KVDataSet.cpp:83
Extension of TDatime to handle various useful date formats.
Definition: KVDatime.h:33
Base class to describe database of an experiment ,,.
Definition: KVExpDB.h:20
virtual void cd()
Definition: KVExpDB.cpp:576
static KVExpDB * MakeDataBase(const Char_t *name, const Char_t *datasetdir)
Definition: KVExpDB.cpp:598
Extended TList class which owns its objects by default.
Definition: KVList.h:28
Description of an individual data file in an experimental dataset.
Definition: KVRunFile.h:19
const Char_t * GetVersion() const
Definition: KVRunFile.h:84
const KVDatime & GetFileWrittenDatime() const
Definition: KVRunFile.h:76
const run_index_t & GetRunIndex() const
Definition: KVRunFile.h:56
KaliVeda extensions to ROOT collection classes.
TObject * At(Int_t idx) const override
Extension of ROOT TString class which allows backwards compatibility with ROOT v3....
Definition: KVString.h:73
void Begin(TString delim) const
Definition: KVString.cpp:565
Bool_t End() const
Definition: KVString.cpp:634
KVString Next(Bool_t strip_whitespace=kFALSE) const
Definition: KVString.cpp:695
static TClass * GetClass(Bool_t load=kTRUE, Bool_t silent=kFALSE)
virtual Int_t GetEntries() const
const char * AsSQLString() const
virtual Bool_t cd()
virtual const char * GetValue(const char *name, const char *dflt) const
virtual void SetValue(const char *name, const char *value, EEnvLevel level=kEnvChange, const char *type=nullptr)
Bool_t Defined(const char *name) const
TObject * FindObject(const char *name) const override
virtual void SetTitle(const char *title="")
const char * GetName() const override
const char * GetTitle() const override
virtual void SetName(const char *name)
TString & String()
const char * Data() const
void ToUpper()
void Form(const char *fmt,...)
TString & Remove(EStripType s, char c)
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
TString & ReplaceAll(const char *s1, const char *s2)
Bool_t cd(const char *path)
const char * pwd()
virtual int Chmod(const char *file, UInt_t mode)
virtual const char * DirName(const char *pathname)
virtual char * ConcatFileName(const char *dir, const char *name)
virtual int mkdir(const char *name, Bool_t recursive=kFALSE)
virtual Int_t Exec(const char *shellcmd)
virtual Bool_t AccessPathName(const char *path, EAccessMode mode=kFileExists)
virtual const char * BaseName(const char *pathname)
virtual Bool_t IsAbsoluteFileName(const char *dir)
Specifies a runfile according to run number and file index ,.
Definition: run_index.h:31
int run() const
Definition: run_index.h:50
TString as_string() const
Definition: run_index.h:95
int index(int no_index=-1) const
Definition: run_index.h:55
bool has_index() const
Definition: run_index.h:59
const Int_t n
void Error(const char *location, const char *fmt,...)
void Info(const char *location, const char *fmt,...)
void Warning(const char *location, const char *fmt,...)
double min(double x, double y)
double max(double x, double y)
const char * String
TArc a
ClassImp(TPyArg)