KaliVeda
Toolkit for HIC analysis
KVDataSet.cpp
1 /*
2 $Id: KVDataSet.cpp,v 1.41 2009/03/11 14:22:41 franklan Exp $
3 $Revision: 1.41 $
4 $Date: 2009/03/11 14:22:41 $
5 $Author: franklan $
6 */
7 
8 #include "TMethodCall.h"
9 #include "KVDataSet.h"
10 #include "KVDataRepository.h"
11 #include "KVDataRepositoryManager.h"
12 #include "KVDataSetManager.h"
13 #include "TSystem.h"
14 #include "TObjArray.h"
15 #include "TObjString.h"
16 #include "KVDBSystem.h"
17 #include "KVDBRun.h"
18 #include "TEnv.h"
19 #include "KVAvailableRunsFile.h"
20 #include "KVNumberList.h"
21 #include "TPluginManager.h"
22 #include "TClass.h"
23 #include "KVRunFile.h"
24 
25 using namespace std;
26 
28 
29 KVDataSet* gDataSet;
30 
31 
34 
36 {
37  //Default constructor
38  fRepository = nullptr;
39  fDataBase = nullptr;
40  fAllTasks.SetOwner(kTRUE);
41  fTasks.SetOwner(kFALSE);
42 }
43 
44 
45 
50 
52 {
53  // \param type name of data type ('raw', 'recon', ...)
54  // \returns pointer to available runs file object for given data type
55  // \note if no data repository is associated with dataset, returns nullptr
56  if (!fRepository) return nullptr;
57  KVAvailableRunsFile* avrf =
58  (KVAvailableRunsFile*) fAvailableRuns.FindObjectByName(type);
59  if (!avrf) {
60  avrf = fRepository->NewAvailableRunsFile(type, this);
61  fAvailableRuns.Add(avrf);
62  }
63  return avrf;
64 }
65 
66 
67 
82 
84 {
85  //Returns name of file containing database for dataset.
86  //
87  //This is fixed as `DataBase.root.M.mm` where
88  // - `M` is the current major version number
89  // - `m` is the current minor version number
90  //
91  //(fixed to be consistent with CMake function INSTALL_KALIVEDA_DATASETS which sets up
92  //the Makefile for each dataset which automatically triggers rebuilding of the ROOT file
93  //when source files change: thus we must have the right file name!)
94  //
95  //This in order to avoid problems if several different versions of KaliVeda
96  //use the same working directory (in user's `$HOME/.kaliveda`) to write the
97  //database files, which are often incompatible between versions
98 
99  TString n = "DataBase.root";
100  n.Append(Form(".%d.%d", GetKVMajorVersion(),
101  GetKVMinorVersion()));
102  return n;
103 }
104 
105 
106 
111 
113 {
114  //\returns name of database object in database file.
115  //
116  //If this is not set explicitly with SetDBName(), we use the name of the dataset by default
117 
118  return (fDBName != "" ? fDBName.Data() : GetName());
119 }
120 
121 
122 
123 
126 
127 void KVDataSet::OpenDBFile(const Char_t* full_path_to_dbfile) const
128 {
129  //Open the database from a file on disk.
130 
131  TDirectory* work_dir = gDirectory; //keep pointer to current directory
132  fDBase.reset(new TFile(full_path_to_dbfile, "READ"));
133 
134  if (fDBase->IsOpen()) {
135  fDataBase = dynamic_cast<KVExpDB*>(fDBase->Get(GetDBName()));
136  if (!fDataBase) {
137  Error("OpenDBFile", "%s not found in file %s", GetDBName().Data(),
138  GetDBFileName().Data());
139  }
140  else {
141  fDataBase->ReadObjects(fDBase.get()); // read any associated objects
142  }
143  work_dir->cd(); //back to initial working directory
144  }
145 }
146 
147 
148 
149 
152 
154 {
155  // \returns full path to file where database is written on disk
156 
157  TString dbfile = GetDBFileName();
158  TString dbfile_fullpath;
159  TString tmp;
160 
161  // If this dataset is just an alias for another dataset i.e. if DataSet.Directory
162  // is set with just the name of another dataset (not a full path to dataset files)
163  // then the database file should be written/found under the name of the alias.
164  TString dataset_alias = GetDataSetEnv("DataSet.Directory", GetName());
165  TString db_alias = GetName();
166  if (!gSystem->IsAbsoluteFileName(dataset_alias)) db_alias = dataset_alias;
167 
168  AssignAndDelete(tmp, gSystem->ConcatFileName(GetDATABASEFilePath(), db_alias.Data()));
169  AssignAndDelete(dbfile_fullpath, gSystem->ConcatFileName(tmp.Data(), dbfile.Data()));
170  return dbfile_fullpath;
171 }
172 
173 
174 
177 
179 {
180  //Check configuration variables to see if the task parameters have been "tweaked" for the dataset.
181 
182  KVString envar = GetDataSetEnv(Form("%s.DataAnalysisTask.Title", t->GetName()));
183  if (envar != "") t->SetTitle(envar);
184  envar = GetDataSetEnv(Form("%s.DataAnalysisTask.Analyser", t->GetName()));
185  if (envar != "") t->SetDataAnalyser(envar);
186  envar = GetDataSetEnv(Form("%s.DataAnalysisTask.UserClass.Base", t->GetName()));
187  if (envar != "") t->SetUserBaseClass(envar);
188  envar = GetDataSetEnv(Form("%s.DataAnalysisTask.Prereq", t->GetName()));
189  if (envar != "") t->SetPrereq(envar);
190  envar = GetDataSetEnv(Form("%s.DataAnalysisTask.UserClass.ExtraACliCIncludes", t->GetName()));
191  if (envar != "") t->SetExtraAClicIncludes(envar);
192  Int_t nev = (Int_t)GetDataSetEnv(Form("%s.DataAnalysisTask.StatusUpdateInterval", t->GetName()), 0.0);
193  if (nev > 0) t->SetStatusUpdateInterval(nev);
194 }
195 
196 
197 
198 
202 
204 {
205  // Write the experiment database for this dataset to disk (ROOT file), creating and setting
206  // permissions for any required directories
207 
208  TString dbfile_fullpath = GetFullPathToDB();
209  TString tmp = gSystem->DirName(dbfile_fullpath.Data()); //full path to directory $KVROOT/db/[dataset name]
210 
211  if (gSystem->AccessPathName(tmp.Data())) { // directory $KVROOT/db/[dataset name] does not exist
212 
213  if (gSystem->mkdir(tmp.Data()) == -1) { // problem creating $KVROOT/db/[dataset name]
214 
215  TString tmp2 = gSystem->DirName(tmp.Data());// full path to directory $KVROOT/db
216 
217  if (gSystem->AccessPathName(tmp2.Data())) { // directory $KVROOT/db does not exist
218 
219  if (gSystem->mkdir(tmp2.Data()) == -1) { // problem creating $KVROOT/db
220  Error("SaveDataBase", "Cannot create directory %s required to save database",
221  tmp2.Data());
222  return;
223  }
224  gSystem->Chmod(tmp2.Data(), 0775);
225  }
226  else {
227  Error("SaveDataBase", "Cannot create directory %s required to save database, even though %s exists: check disk space ?",
228  tmp.Data(), tmp2.Data());
229  return;
230  }
231  //try again
232  if (gSystem->mkdir(tmp.Data()) == -1) {
233  Error("SaveDataBase", "Cannot create directory %s required to save database",
234  tmp.Data());
235  return;
236  }
237  else {
238  gSystem->Chmod(tmp.Data(), 0775);
239  }
240  }
241  else {
242  gSystem->Chmod(tmp.Data(), 0775);
243  }
244  }
245 
246  WriteDBFile(dbfile_fullpath.Data());
247 }
248 
249 
250 
251 
254 
255 void KVDataSet::WriteDBFile(const Char_t* full_path_to_dbfile) const
256 {
257  //Write the database to disk.
258 
259  TDirectory* work_dir = gDirectory; //keep pointer to current directory
260  if (!fDataBase) {
261  Error("WriteDBFile", "Database has not been built");
262  return;
263  }
264  fDBase.reset(new TFile(full_path_to_dbfile, "recreate"));
265  fDBase->cd(); //set as current directory (maybe not necessary)
266  fDataBase->Write(GetDBName()); //write database to file with given name
267  fDataBase->WriteObjects(fDBase.get()); //write any associated objects
268  fDBase->Write(); // write file header etc.
269  fDBase->Close(); // close file
270  gSystem->Chmod(full_path_to_dbfile, 0664); // set permissions to rw-rw-r--
271  work_dir->cd(); //back to initial working directory
272 }
273 
274 
275 
284 
286 {
287  //\returns pointer to database associated with this dataset.
288  //
289  //Opens, updates or creates database file if necessary
290  //(the database is automatically rebuilt if the source files are
291  //more recent than the last database file).
292  //
293  //\param[in] opt if opt="update": force regeneration of the database from source files in dataset directory
294 
295  TString _opt(opt);
296  _opt.ToUpper();
297  if (_opt == "UPDATE") {
298  OpenDataBase(_opt.Data());
299  }
300  else {
301  OpenDataBase();
302  }
303  return fDataBase;
304 }
305 
306 
307 
318 
320 {
321  //Open the database for this dataset.
322  //
323  //If the database does not exist or is older than the source files
324  //the database is automatically rebuilt (see DataBaseNeedUpdate()).
325  //
326  //\param[in] opt if opt="UPDATE" we force rebuilding of the database
327  //
328  //\warning if the database needs to be (re)built, we set gDataSet to point to this dataset in case it was not already done,
329  //as in order to (re)build the database it may be necessary for gDataSet to point to the current dataset.
330 
331  Bool_t is_glob_db = kFALSE;
332  //if option="update" or database out of date or does not exist, (re)build the database
333  if ((!strcmp(opt, "UPDATE")) || DataBaseNeedsUpdate()) {
334  //check if it is the currently active database (gDataBase),
335  //in which case we must 'cd()' to it after rebuilding
336  Info("OpenDataBase", "Updating database file");
337  fDataBaseUpdateInProgress = true;
338  is_glob_db = (fDataBase == gExpDB);
339  if (fDataBase) {
340  delete fDataBase;
341  fDataBase = 0;
342  }
343  // make sure gDataSet is set & points to us
344  gDataSet = const_cast<KVDataSet*>(this);
345  fDataBase = KVExpDB::MakeDataBase(GetDBName(), GetDataSetDir());
346  if (!fDataBase) {
347  // no database defined for dataset
348  Info("OpenDataBase", "No database defined for dataset");
349  return;
350  }
351  SaveDataBase();
352  if (fDataBase && is_glob_db) fDataBase->cd();
353  fDataBaseUpdateInProgress = false;
354  }
355  else if (!fDataBase) {
356  // if database is not in memory at this point, we need to
357  // open the database file and read in the database
358 
359  //load plugin for database
360  if (!LoadPlugin("KVExpDB", GetDBName())) {
361  Error("GetDataBase", "Cannot load required plugin library");
362  return;
363  }
364  //look for database file in dataset subdirectory
365  TString dbfile_fullpath = GetFullPathToDB();
366  //open database file
367  OpenDBFile(dbfile_fullpath.Data());
368  }
369 }
370 
371 
372 
373 
376 
378 {
379  //Print dataset information
380  cout << "Dataset name=" << GetName() << " (" << GetTitle() << ")";
381  if (IsAvailable()) {
382  cout << " [ AVAILABLE: ";
383  cout << fDatatypes.Data();
384  cout << "]";
385  }
386  else
387  cout << " [UNAVAILABLE]";
388  cout << endl;
389 }
390 
391 
392 
399 
400 void KVDataSet::Print(Option_t* opt) const
401 {
402  //Print dataset information
403  //
404  //param[in] opt select optional output formats:
405  // - if string contains "tasks", print numbered list of tasks that can be performed
406  // - if string contains "data", print list of available data types
407 
408  TString Sopt(opt);
409  Sopt.ToUpper();
410  if (Sopt.Contains("TASK")) {
411  if (!GetNtasks()) {
412  cout << " *** No available analysis tasks ***"
413  << endl;
414  return;
415  }
416  else {
417  for (int i = 1; i <= GetNtasks(); i++) {
418  KVDataAnalysisTask* dat = GetAnalysisTask(i);
419  cout << "\t" << i << ". " << dat->GetTitle() << endl;
420  }
421  }
422  cout << endl;
423  }
424  else if (Sopt.Contains("DATA")) {
425  cout << "Available data types: " << fDatatypes.Data() << endl;
426  }
427  else {
428  ls(opt);
429  }
430 }
431 
432 
433 
442 
444 {
445  //Check if this data set is physically present and available for analysis.
446  //
447  //In other words we check if the value of GetDataPathSubdir() is a subdirectory
448  //of the current data repository.
449  //If so, we proceed to check for the existence of sudirectories corresponding
450  // to any of the datatypes associated with the dataset.
451  //
452 
453  if (!fRepository) // for a stand-alone KVDataSetManager not linked to a KVDataRepository,
454  SetAvailable(); // all known datasets are 'available'
455  else
456  SetAvailable(fRepository->CheckSubdirExists(GetDataPathSubdir()));
457  if (!IsAvailable())
458  return;
459  //check subdirectories
460  KVString data_types = GetDataSetEnv("KVDataSet.DataTypes", "");
461  if (data_types == "") {
462  Warning("CheckAvailable", "No datatypes defined for this dataset: %s\nCheck value of KVDataSet.DataTypes or %s.KVDataSet.DataTypes",
463  GetName(), GetName());
464  SetAvailable(kFALSE);
465  }
466  fDatatypes = "";
467  // loop over data types
468  data_types.Begin(" ");
469  while (!data_types.End()) {
470  KVString type = data_types.Next(kTRUE);
471  if (!fRepository ||
472  (fRepository && fRepository->CheckSubdirExists(GetDataPathSubdir(), GetDataTypeSubdir(type.Data())))
473  ) {
474  AddAvailableDataType(type.Data());
475  }
476  }
477  //check at least one datatype exists
478  SetAvailable(fDatatypes != "");
479  //check user name against allowed groups
480  if (!CheckUserCanAccess()) {
481  SetAvailable(kFALSE);
482  return;
483  }
484 }
485 
486 
487 
489 
491 {
492  if (fDatatypes != "") fDatatypes += " ";
493  KVString _type = type;
494  _type.Remove(TString::kBoth, ' '); //strip whitespace
495  fDatatypes += _type;
496 }
497 
498 
499 
506 
508 {
509  // Add to fAllTasks list any data analysis task in list 'task_list'
510  //
511  // Add to fTasks list any data analysis task in list 'task_list' whose pre-requisite datatype is present for this dataset.
512  //
513  // Any dataset-specific "tweaking" of the task (including the prerequisite datatype) is done here.
514 
515  TString availables = gEnv->GetValue(Form("%s.DataAnalysisTask", GetName()), "");
516  fAllTasks.Clear();
517  fTasks.Clear();
518  TIter nxt(task_list);
519  KVDataAnalysisTask* dat;
520  while ((dat = (KVDataAnalysisTask*) nxt())) {
521  //make new copy of default analysis task
522  if (availables == "" || availables.Contains(dat->GetName())) {
523  KVDataAnalysisTask* new_task = new KVDataAnalysisTask(*dat);
524  //check if any dataset-specific parameters need to be changed
525  SetDataSetSpecificTaskParameters(new_task);
526  fAllTasks.Add(new_task);
527  // add tasks with available prerequisite data to fTasks
528  if (HasDataType(new_task->GetPrereq())) {
529  fTasks.Add(new_task);
530  }
531  }
532  }
533 }
534 
535 
536 
540 
542 {
543  //Returns the number of tasks associated to dataset which are compatible
544  //with the available data
545 
546  return fTasks.GetSize();
547 }
548 
549 
550 
555 
557 {
558  //Return kth analysis task in list of available tasks.
559  //
560  //\param[in] k task number in range [1, GetNtasks()]. Corresponds to the number shown next to the title of the task when Print("tasks") is called
561  return (KVDataAnalysisTask*) fTasks.At(k - 1);
562 }
563 
564 
565 
572 
573 std::unique_ptr<TList> KVDataSet::GetListOfAvailableSystems(const Char_t* datatype, KVDBSystem* systol)
574 {
575  //\returns list of available systems or runfiles for this dataset and the given datatype
576  //
577  //\param[in] systol when systol=nullptr (default), return list of available systems. if pointer to system, return list of available runfiles for the given system
578  //
579  //If no systems are defined for the dataset then we return a list of available runfiles for the given datatype
580 
581  if (!GetAvailableRunsFile(datatype)) {
582  Error("GetListOfAvailableSystems(const Char_t*)",
583  "No available runs file for type %s", datatype);
584  return 0;
585  }
586  return GetAvailableRunsFile(datatype)->GetListOfAvailableSystems(systol);
587 }
588 
589 
590 
597 
599 {
600  //\returns list of available systems or runfiles for this dataset and the prerequisite datatype for the given analysis task
601  //
602  //\param[in] systol when systol=nullptr (default), return list of available systems. if pointer to system, return list of available runfiles for the given system
603  //
604  //If no systems are defined for the dataset then we return a list of available runfiles for the given datatype
605 
606  return GetListOfAvailableSystems(datan->GetPrereq(), systol);
607 }
608 
609 
610 
631 
632 void KVDataSet::SetName(const char* name)
633 {
634  // Set name of dataset
635  //
636  // Also sets path to directory containing database informations
637  // for this dataset, i.e. list of runs, systems, calibration files etc.
638  //
639  // By default, just the name of the dataset is used, i.e.
640  // `[DATADIR]/name`
641  // (where `DATADIR` = path given by KVBase::GetDATADIRFilePath())
642  //
643  // However, if the variable
644  //~~~
645  // [name].DataSet.Directory: [path]
646  //~~~
647  // has been set, the value of `[path]` will be used:
648  // - if [path] is an absolute path name, it will be used as such
649  // - if [path] is an incomplete or relative path, it will be prepended with `[DATADIR]/`
650  //
651  // This allows to use one dataset as an alias for another, by setting `DataSet.Directory`
652  // to the name of an existing dataset
654  TString path = GetDataSetEnv("DataSet.Directory", name);
655  if (gSystem->IsAbsoluteFileName(path)) fCalibDir = path;
656  else {
657  // in this case (not an absolute path but just the name of another dataset)
658  // this dataset is an alias for another dataset.
659  fCalibDir = GetDATADIRFilePath(path);
660  // the name of the database object is the name of the "true" dataset
661  SetDBName(path);
662  }
663 }
664 
665 
666 
669 
671 {
672  //\returns full path to directory containing database and calibration/identification parameters etc. for this dataset.
673 
674  return fCalibDir.Data();
675 }
676 
677 
678 
691 
692 void KVDataSet::cd() const
693 {
694  // Makes this dataset the "currently active" or default dataset.
695  //
696  //At the same time, the database and data repository associated with
697  //this dataset also become the "currently active" ones:
698  //
699  // | global pointer | represents | base class |
700  // |----------------|------------|------------|
701  // | `gDataSet` | active dataset | KVDataSet |
702  // | `gExpDB` | associated experimental database | KVExpDB |
703  // | `gDataRepository` | repository containing runfiles | KVDataRepository |
704  //
705 
706  gDataSet = const_cast<KVDataSet*>(this);
707  if (fRepository) fRepository->cd();
708  KVExpDB* db = GetDataBase();
709  if (db) db->cd();
710 }
711 
712 
713 
741 
743 {
744  // Open file containing data of given datatype for given run number of this dataset.
745  //
746  // \returns a pointer to the opened file; if the file is not available, we return nullptr.
747  //
748  // The user must cast the returned pointer to the correct class, which will
749  // depend on the data type and the dataset
750  //
751  // **SPECIAL CASE: MFM data with EBYEDAT frames**
752  //
753  // If the variable
754  //
755  //~~~~~~~~~~~~~~~~~~~~~~~~~
756  // [dataset].MFM.WithEbyedat: yes
757  //~~~~~~~~~~~~~~~~~~~~~~~~~
758  //
759  // is set, then we expect to find the necessary `ACTIONS_*` files in the dataset directory
760  // in subdirectory `ebyedat` (they should have the same names as the data files prefixed by
761  // `ACTIONS_[expname].CHC_PAR.`).
762  //
763  // If in addition the variable
764  //
765  //~~~~~~~~~~~~~~~~~~~~~~~~~
766  // [dataset].MFM.EbyedatActionsExpName: [expname]
767  //~~~~~~~~~~~~~~~~~~~~~~~~~
768  //
769  // is set, then we use the same `ACTIONS` file for all runs, with name `ACTIONS_[expname].CHC_PAR`
770 
771 
772  if (!strcmp(type, "raw") && !strcmp(GetDataSetEnv("MFM.WithEbyedat", ""), "yes")) {
773  TString ebydir = GetDataSetDir();
774  ebydir += "/ebyedat";
775  gEnv->SetValue("KVMFMDataFileReader.ActionsDirectory", ebydir);
776  if (strcmp(GetDataSetEnv("MFM.EbyedatActionsExpName", ""), ""))
777  gEnv->SetValue("KVMFMDataFileReader.ActionsExpName", GetDataSetEnv("MFM.EbyedatActionsExpName", ""));
778  TObject* f = GetRepository()->OpenDataSetRunFile(this, type, run, GetName());
779  // reset in case another dataset opens a raw MFM file without EBYEDAT data
780  gEnv->SetValue("KVMFMDataFileReader.ActionsDirectory", "");
781  gEnv->SetValue("KVMFMDataFileReader.ActionsExpName", "");
782  return f;
783  }
784  return GetRepository()->OpenDataSetRunFile(this, type, run, GetName());
785 }
786 
787 
788 
789 
793 
795  const run_index_t& run) const
796 {
797  //\return full path to file containing data of given datatype for given run/index of this dataset
798  // \note only works for available run files, if their is no file in the repository for this run, the returned path will be empty
799 
800  TString file("");
801  if (fRepository) file = GetRunfileName(type, run);
802  if (file == "")
803  return file.Data();
804  return fRepository->GetFullPathToOpenFile(this, type, file.Data());
805 }
806 
807 
808 
809 
813 
815 {
816  //\return name of file containing data of given datatype for given run/index of this dataset
817  //\note only works for available run files, if there is no file in the repository for this run, the returned path will be empty
818 
819  if (!HasDataType(type)) {
820  Error("GetRunfileName",
821  "No data of type \"%s\" available for dataset %s", (const char*)type,
822  GetName());
823  return 0;
824  }
825  //get name of file from available runs file
826  return GetAvailableRunsFile(type)->GetFileName(run);
827 }
828 
829 
830 
831 
837 
838 std::optional<TDatime> KVDataSet::GetRunfileDate(const KVString& type, const run_index_t& run)
839 {
840  //\return date of file containing data of given datatype for given run/index of this dataset
841  //
842  //\note only works for available runfiles, if there is no file in the repository for this run/index,
843  //an error will be printed and std::optional will not contain a value
844 
845  if (!HasDataType(type)) {
846  Error("GetRunfileDate",
847  "No data of type \"%s\" available for dataset %s", (const char*)type,
848  GetName());
849  return std::nullopt;
850  }
851  //get date of file from available runs file
852  TDatime date;
854  if (!GetAvailableRunsFile(type)->GetRunInfo(run, date, filename)) {
855  Error("GetRunfileDate",
856  "Runfile not found for run %d index %d (data type: %s)", run.run(), run.index(), (const char*)type);
857  return std::nullopt;
858  }
859  return date;
860 }
861 
862 
863 
886 
887 std::optional<run_index_t> KVDataSet::GetRunNumberFromFileName(const TString& datatype, const TString& filename)
888 {
889  // \returns run number and eventual index by parsing the name of a runfile for given datatype. if filename does not have correct format, returns std::nullopt
890  //
891  // If the runfile in question was generated using an index multiplier, e.g. if it contains the number
892  // '17001' when the index multiplier defined for the dataset is 1000, we return '17' as the
893  // run number and '1' as the index.
894  //
895  // Usage examples (using global pointer gDataSet to currently 'active' dataset):
896  //~~~{.cpp}
897  //auto run = gDataSet->GetRunNumberFromFileName(filename);
898  //
899  //if(!run) // no value (std::nullopt) returned
900  //{
901  // std::cerr << "Filename " << filename << " is not right format\n";
902  //}
903  //else
904  //{
905  // std::cout << "Filename " << filename << " corresponds to run number " << run.value().run();
906  // if(run.has_index()) std::cout << ", file index = " << run.index();
907  // std::cout << std::endl;
908  //}
909  //~~~
910 
911  auto arf = GetAvailableRunsFile(datatype);
912  auto v = arf->IsRunFileName(filename);
913  if (!v) return v;
914  // if the returned value has a non-null index, there is nothing more to do
915  if (v.value().has_index()) return v;
916  int r = v.value().run();
917  // check if returned run number actually consists of run*multiplier+index
918  if (GetDataBase()->HasIndexMultiplier() && r > GetDataBase()->GetIndexMultiplier()) {
919  r /= GetDataBase()->GetIndexMultiplier();
920  int i = v.value().run() % GetDataBase()->GetIndexMultiplier();
921  return run_index_t{r, i};
922  }
923  return v;
924 }
925 
926 
927 
928 
933 
935 {
936  //We check the availability of the run by looking in the available runs file associated
937  //with the given datatype.
938 
939  //check data type is available
940  if (!HasDataType(type)) {
941  Error("CheckRunfileAvailable",
942  "No data of type \"%s\" available for dataset %s", (const char*)type,
943  GetName());
944  return 0;
945  }
946  return GetAvailableRunsFile(type)->CheckAvailable(run);
947 }
948 
949 
950 
951 
966 
967 const Char_t* KVDataSet::GetBaseFileName(const Char_t* type, const run_index_t& run) const
968 {
969  //PRIVATE METHOD: Returns base name of data file containing data for the run of given datatype.
970  //The filename corresponds to one of the formats defined in $KVROOT/KVFiles/.kvrootrc
971  //by variables like:
972  //
973  //~~~
974  //[dataset].DataSet.RunFileName.[type]: run%R.dat
975  //~~~
976  //
977  //%R will be replaced with the run number
978  //
979  //IF the format contains '%D' it will be replaced with the current date and time
980  //
981  // Any index will be appended at the end: ".index"
982 
983  static TString tmp;
984  //get format string
985  TString fmt = GetDataSetEnv(Form("DataSet.RunFileName.%s", type));
986  TString run_num(Form("%d", run.run()));
987  KVDatime now;
988  TString date(now.AsSQLString());
989  tmp = fmt;
990  tmp.ReplaceAll("%R", run_num);
991  if (fmt.Contains("%D")) {
992  tmp.ReplaceAll("%D", date);
993  }
994  if (run.has_index()) tmp += Form(".%d", run.index());
995  return tmp.Data();
996 }
997 
998 
999 
1000 
1004 
1006 {
1007  //Update list of available runs for given data 'type'
1008 
1009  //check data type is available
1010  if (!HasDataType(type)) {
1011  Error("UpdateAvailableRuns",
1012  "No data of type \"%s\" available for dataset %s", (const char*)type,
1013  GetName());
1014  }
1015  KVAvailableRunsFile* a = GetAvailableRunsFile(type);
1016  a->Update();
1017 }
1018 
1019 
1020 
1021 
1026 
1028 {
1029  // Create a new runfile for the dataset of given datatype.
1030  // (only if this dataset is associated with a data repository)
1031  // Once the file has been filled, use CommitRunfile to submit it to the repository.
1032 
1033  if (!fRepository) return nullptr;
1034  TString tmp = GetBaseFileName(type, run);
1035  //turn any spaces into "_"
1036  tmp.ReplaceAll(" ", "_");
1037  return fRepository->CreateNewFile(this, type, tmp.Data());
1038 }
1039 
1040 
1041 
1042 
1051 
1052 void KVDataSet::DeleteRunfile(const KVString& type, const run_index_t& run, Bool_t confirm)
1053 {
1054  // Delete the file for the given run/index of data type "type" from the repository.
1055  // By default, confirm=kTRUE, which means that the user will be asked to confirm
1056  // that the file should be deleted. If confirm=kFALSE, no confirmation will be asked
1057  // for and the file will be deleted straight away.
1058  //
1059  // WARNING: this really does DELETE files in the repository, they cannot be
1060  // retrieved once they have been deleted.
1061 
1062  if (!fRepository) return;
1063 
1064  //get name of file to delete
1065  TString filename = GetAvailableRunsFile(type)->GetFileName(run);
1066  if (filename == "") {
1067  Error("DeleteRunfile", "Run %s of type %s does not exist.",
1068  run.as_string().Data(), (const char*)type);
1069  return;
1070  }
1071  //delete file
1072  //prevent accidental deletion of certain types of runfiles
1073  KVString doNotDelete = GetDataSetEnv("DataSet.RunFile.DoNotDelete", "all");
1074  if (doNotDelete == "all" || doNotDelete.Contains(type)) {
1075  Error("DeleteRunFile", "%s files cannot be deleted", (const char*)type);
1076  return;
1077  }
1078  fRepository->DeleteFile(this, type, filename.Data(), confirm);
1079  //was file deleted ? if so, remove entry from available runs file
1080  if (!fRepository->CheckFileStatus(this, type, filename.Data()))
1081  GetAvailableRunsFile(type)->Remove(run);
1082 }
1083 
1084 
1085 
1086 
1098 
1099 void KVDataSet::DeleteRunfiles(const Char_t* type, const run_index_list& nl, Bool_t confirm)
1100 {
1101  // Delete files corresponding to a list of runs/index of data type "type" from the repository.
1102  //
1103  // By default, confirm=kTRUE, which means that the user will be asked to confirm
1104  // that each file should be deleted.
1105  //
1106  // If confirm=kFALSE, no confirmation will be asked for and the file will be deleted straight away.
1107  //
1108  // if "nl" is empty (default value) all runs of the dataset corresponding to the given type will be deleted
1109  //
1110  // WARNING: this really does DELETE files in the repository, they cannot be retrieved once they have been deleted.
1111 
1112  auto NL = nl;
1113  if (NL.IsEmpty()) NL = GetRunList(type);
1114  if (NL.IsEmpty()) return;
1115  for (auto& r : NL)
1116  DeleteRunfile(type, r, confirm);
1117 }
1118 
1119 
1120 
1121 
1129 
1130 run_index_list KVDataSet::GetRunList_DateSelection(const Char_t* type, TDatime* min, TDatime* max)
1131 {
1132  // Prints out and returns list of runs after date / time selection
1133  //
1134  // Runs generated between ]min;max[ are selected
1135  // - if min=NULL runs with date <max are selected
1136  // - if max=NULL runs with date >min are selected
1137  // - if max and min are NULL returns empty list
1138 
1139  if (!min && !max) return {};
1140 
1141  if (min) printf("date minimum %s\n", min->AsString());
1142  if (max) printf("date maximum %s\n", max->AsString());
1143 
1144  run_index_list numb;
1145 
1146  auto ll = GetListOfAvailableSystems(type);
1147  KVDBSystem* sys = 0;
1148  KVRunFile* run = 0;
1149  for (Int_t nl = 0; nl < ll->GetEntries(); nl += 1) {
1150  sys = (KVDBSystem*)ll->At(nl);
1151  auto lrun = GetListOfAvailableSystems(type, sys);
1152  auto oldList = numb;
1153  for (Int_t nr = 0; nr < lrun->GetEntries(); nr += 1) {
1154  run = (KVRunFile*)lrun->At(nr);
1155 
1156  if (min && max) {
1157  if (*min < run->GetFileWrittenDatime() && run->GetFileWrittenDatime() < *max) {
1158  numb.Add(run->GetRunIndex());
1159  }
1160  }
1161  else if (min) {
1162  if (*min < run->GetFileWrittenDatime()) {
1163  numb.Add(run->GetRunIndex());
1164  }
1165  }
1166  else if (max) {
1167  if (run->GetFileWrittenDatime() < *max) {
1168  numb.Add(run->GetRunIndex());
1169  }
1170  }
1171  }
1172  // print runs for system if any
1173  if (numb.GetEntries() > oldList.GetEntries())
1174  printf("%s : %s\n", sys->GetName(), (numb - oldList).AsString().Data());
1175  }
1176  return numb;
1177 
1178 }
1179 
1180 
1181 
1182 
1191 
1192 run_index_list KVDataSet::GetRunList_StageSelection(const Char_t* type, const Char_t* ref_type, KVDBSystem* system, Bool_t OnlyCol)
1193 {
1194  // Returns list of runs which are present for data type "base_type" but not for "other_type"
1195  //
1196  // if type is NULL or ="" returns empty KVNumberList
1197  //
1198  // If pointer to system is given, only runs for the system are considered.
1199  //
1200  // If OnlyCol=kTRUE (kFALSE default) only systems with KVDBSystem::IsCollision()=kTRUE are considered
1201 
1202  run_index_list manquant;
1203  auto ll = GetListOfAvailableSystems(ref_type);
1204  if (!ll || !ll->GetEntries()) {
1205  //numb.Clear();
1206  Info("GetRunList_StageSelection", "No data available of type \"%s\"", ref_type);
1207  return manquant;
1208  }
1209  if (system && !ll->FindObject(system)) {
1210  Info("GetRunList_StageSelection", "No data available of type \"%s\" for system %s", ref_type, system->GetName());
1211  return manquant;
1212  }
1213 
1214  Info("GetRunList_StageSelection", "Liste des runs presents dans \"%s\" mais absent dans \"%s\"", ref_type, type);
1215 
1216  KVDBSystem* sys = 0;
1217 
1218  for (Int_t nl = 0; nl < ll->GetEntries(); nl += 1) {
1219 
1220  sys = (KVDBSystem*)ll->At(nl);
1221  if (system && sys != system) continue;
1222  if (OnlyCol && !sys->IsCollision()) continue;
1223  auto nsys = GetRunList(type, sys);
1224  auto nsys_ref = GetRunList(ref_type, sys);
1225  Int_t nref = nsys_ref.GetNValues();
1226 
1227  nsys_ref.Remove(nsys);
1228 
1229  Info("GetRunList_StageSelection", "\nKVDBSystem : %s --> %d runs manquants sur %d : %s",
1230  sys->GetName(),
1231  nsys_ref.GetNValues(),
1232  nref,
1233  nsys_ref.AsString().Data()
1234  );
1235  manquant.Add(nsys_ref);
1236  }
1237  return manquant;
1238 }
1239 
1240 
1241 
1245 
1246 run_index_list KVDataSet::GetRunList_VersionSelection(const Char_t* type, const Char_t* version, KVDBSystem* sys)
1247 {
1248  // Returns list of runs of given type that were created with the given version of KaliVeda.
1249  // If system!="" then only runs for the given system are considered
1250 
1251  run_index_list runs;
1252  if (sys) {
1253  unique_ptr<TList> lrun(GetListOfAvailableSystems(type, sys));
1254  TIter next(lrun.get());
1255  KVRunFile* run;
1256  while ((run = (KVRunFile*)next())) {
1257  if (!strcmp(run->GetVersion(), version)) runs.Add(run->GetRunIndex());
1258  }
1259  return runs;
1260  }
1261  unique_ptr<TList> ll(GetListOfAvailableSystems(type));
1262  if (!ll.get() || !ll->GetEntries()) {
1263  //numb.Clear();
1264  Info("GetRunList_VersionSelection", "No data available of type \"%s\"", type);
1265  return runs;
1266  }
1267  Int_t nsys = ll->GetEntries();
1268  for (Int_t nl = 0; nl < nsys; nl += 1) {
1269  sys = (KVDBSystem*)ll->At(nl);
1270  unique_ptr<TList> lrun(GetListOfAvailableSystems(type, sys));
1271  TIter next(lrun.get());
1272  KVRunFile* run;
1273  while ((run = (KVRunFile*)next())) {
1274  if (!strcmp(run->GetVersion(), version)) runs.Add(run->GetRunIndex());
1275  }
1276  }
1277  return runs;
1278 }
1279 
1280 
1281 
1286 
1287 void KVDataSet::CommitRunfile(const KVString& type, const run_index_t& run, TFile* file)
1288 {
1289  // Commit a runfile previously created with NewRunfile() to the repository.
1290  // Any previous version of the runfile will be deleted.
1291  // The available runs list for this data 'type' is updated.
1292 
1293  if (!fRepository) return;
1294 
1295  //keep name of file for updating available runs list
1296  TString newfile = gSystem->BaseName(file->GetName());
1297 
1298  fRepository->CommitFile(file, type, this);
1299  //update list of available datatypes of dataset,
1300  //in case this addition has created a new subdirectory
1301  CheckAvailable();
1302  //check if previous version of file exists
1303  //get name of file from available runs file
1304  //note that when the file is the first of a new subdirectory, GetAvailableRunsFile->GetFileName
1305  //will cause the available runs file to be created, and it will contain one entry:
1306  //the new file!
1307  TString oldfile = GetAvailableRunsFile(type)->GetFileName(run);
1308  if (oldfile != "" && oldfile != newfile) {
1309  //delete previous version - no confirmation
1310  fRepository->DeleteFile(this, type, oldfile.Data(),
1311  kFALSE);
1312  //was file deleted ? if so, remove entry from available runs file
1313  if (!fRepository->CheckFileStatus(this, type, oldfile.Data()))
1314  GetAvailableRunsFile(type)->Remove(run);
1315  }
1316  if (oldfile != newfile) {
1317  //add entry for new run in available runs file
1318  GetAvailableRunsFile(type)->Add(run, newfile.Data());
1319  }
1320 }
1321 
1322 
1323 
1324 
1330 
1332 {
1333  //if fUserGroups has been set with SetUserGroups(), we check that the current user's name
1334  //(gSystem->GetUserInfo()->fUser) appears in at least one of the groups in the list.
1335  //Returns kFALSE if user's name is not found in any of the groups.
1336  //if fUserGroups="" (default), we return kTRUE for all users.
1337 
1338  if (fUserGroups == "")
1339  return kTRUE; /* no groups set, all users have access */
1340 
1341  //split into array of group names
1342  unique_ptr<TObjArray> toks(fUserGroups.Tokenize(' '));
1343  TObjString* group_name;
1344  TIter next_name(toks.get());
1345  while ((group_name = (TObjString*) next_name())) {
1346  //for each group_name, we check if the user's name appears in the group
1347  if (!fRepository || (fRepository && fRepository->GetDataSetManager()->
1348  CheckUser(group_name->String().Data()))
1349  ) {
1350  return kTRUE;
1351  }
1352  }
1353  return kFALSE;
1354 }
1355 
1356 
1357 
1358 
1361 
1363 {
1364  //Set pointer to data repository in which dataset is stored
1365  fRepository = dr;
1366 }
1367 
1368 
1369 
1370 
1373 
1375 {
1376  //Get pointer to data repository in which dataset is stored
1377  return fRepository;
1378 }
1379 
1380 
1381 
1382 
1387 
1389 {
1390  //Check all runs for a given datatype and make sure that only one version
1391  //exists for each runfile. If not, we print a report on the runfiles which occur
1392  //multiple times, with the associated date and file name.
1393 
1394  auto doubles = GetAvailableRunsFile(data_type)->CheckMultiRunfiles();
1395  if (doubles.empty()) {
1396  cout << "OK. No runs appear more than once." << endl;
1397  }
1398  else {
1399  cout << "Runs which appear more than once: " << endl << endl;
1400  //print dates and filenames for each run
1401 
1402  for (auto& rr : doubles) {
1403  KVList filenames, dates;
1404 
1405  //get infos for current run
1406  GetAvailableRunsFile(data_type)->GetRunInfos(rr, &dates, &filenames);
1407 
1408  cout << "Run " << rr << " : " << dates.GetEntries() << " files >>>>>>" << endl;
1409  for (int i = 0; i < dates.GetEntries(); i++) {
1410 
1411  cout << "\t" << ((TObjString*) filenames.At(i))->String().
1412  Data() << "\t" << ((TObjString*) dates.At(i))->String().
1413  Data() << endl;
1414 
1415  }
1416  }
1417  }
1418 }
1419 
1420 
1421 
1422 
1431 
1432 void KVDataSet::CleanMultiRunfiles(const Char_t* data_type, Bool_t confirm)
1433 {
1434  // Check all runs for a given datatype and make sure that only one version
1435  // exists for each run. If not, we print a report on the runfiles which occur
1436  // multiple times, with the associated date and file name, and then we
1437  // destroy all but the most recent version of the file in the repository, and
1438  // update the runlist accordingly.
1439  // By default, we ask for confirmation before deleting each file.
1440  // Call with confirm=kFALSE to delete WITHOUT CONFIRMATION (DANGER!! WARNING!!!)
1441 
1442  if (!fRepository) return;
1443 
1444  KVAvailableRunsFile* ARF = GetAvailableRunsFile(data_type);
1445  auto doubles = ARF->CheckMultiRunfiles();
1446  if (doubles.empty()) {
1447  cout << "OK. No runs appear more than once." << endl;
1448  }
1449  else {
1450  cout << "Runs which appear more than once: " << endl << endl;
1451  //print dates and filenames for each run
1452 
1453  KVList filenames, dates;
1454  for (auto& rr : doubles) {
1455  //get infos for current run
1456  ARF->GetRunInfos(rr, &dates, &filenames);
1457  TDatime most_recent("1998-12-25 00:00:00");
1458  Int_t i_most_recent = 0;
1459  cout << "Run " << rr << " : " << dates.GetEntries() << " files >>>>>>" << endl;
1460  for (int i = 0; i < dates.GetEntries(); i++) {
1461  //check if run is most recent
1462  TDatime rundate(((TObjString*) dates.At(i))->String().Data());
1463  if (rundate > most_recent) {
1464  most_recent = rundate;
1465  i_most_recent = i;
1466  }
1467  }
1468  //Now, we loop over the list again, this time we destroy all but the most recent
1469  //version of the runfile
1470  for (int i = 0; i < dates.GetEntries(); i++) {
1471  if (i == i_most_recent) {
1472  cout << "KEEP : ";
1473  }
1474  else {
1475  cout << "DELETE : ";
1476  }
1477  cout << "\t" << ((TObjString*) filenames.At(i))->String().
1478  Data() << "\t" << ((TObjString*) dates.At(i))->String().
1479  Data() << endl;
1480  if (i != i_most_recent) {
1481  //delete file from repository forever and ever
1482  fRepository->DeleteFile(this, data_type,
1483  ((TObjString*) filenames.At(i))->
1484  String().Data(), confirm);
1485  //remove file entry from available runlist
1486  ARF->Remove(rr,
1487  ((TObjString*) filenames.At(i))->String());
1488  }
1489  }
1490  }
1491  }
1492 }
1493 
1494 
1495 
1496 
1502 
1504  KVDataRepository* other_repos)
1505 {
1506  //Use this method to check whether the file of type "data_type" for run number "run"
1507  //in the data repository "other_repos" is more recent than the file contained in the data
1508  //repository corresponding to this dataset.
1509  //Returns kFALSE if file in other repository is more recent.
1510 
1511  if (!other_repos)
1512  return kTRUE;
1513  //get dataset with same name as this one from dataset manager of other repository
1514  KVDataSet* ds = other_repos->GetDataSetManager()->GetDataSet(GetName());
1515  if (!ds) {
1516  Error("CheckRunfileUpToDate",
1517  "Dataset \"%s\" not found in repository \"%s\"", GetName(),
1518  other_repos->GetName());
1519  return kFALSE;
1520  }
1521  //compare dates of the two runfiles
1522  if (GetRunfileDate(data_type, run) < ds->GetRunfileDate(data_type, run))
1523  return kFALSE;
1524  return kTRUE;
1525 }
1526 
1527 
1528 
1529 
1536 
1538  const KVString& other_repos)
1539 {
1540  //Use this method to check whether the file of type "data_type" for run number "run"
1541  //in the data repository "other_repos" is more recent than the file contained in the data
1542  //repository corresponding to this dataset.
1543  //Returns kTRUE if no repository with name "other_repos" exists.
1544  //Returns kFALSE if file in other repository is more recent.
1545 
1546  KVDataRepository* _or =
1547  gDataRepositoryManager->GetRepository(other_repos);
1548  if (_or)
1549  return CheckRunfileUpToDate(data_type, run, _or);
1550  Error("CheckRunfileUpToDate",
1551  "No data repository known with this name : %s", (const char*) other_repos);
1552  return kTRUE;
1553 }
1554 
1555 
1556 
1557 
1561 
1562 void KVDataSet::CheckUpToDate(const Char_t* data_type,
1563  const Char_t* other_repos)
1564 {
1565  //Check whether all files of type "data_type" for run number "run" in the data repository
1566  //are up to date (i.e. at least as recent) as compared to the files in data repository "other_repos".
1567 
1568  if (!fRepository) return;
1569 
1570  KVDataRepository* _or =
1571  gDataRepositoryManager->GetRepository(other_repos);
1572  if (!_or) {
1573  Error("CheckUpToDate",
1574  "No data repository known with this name : %s", other_repos);
1575  return;
1576  }
1577  auto runlist = GetAvailableRunsFile(data_type)->GetRunList();
1578  Int_t need_update = 0;
1579  for (auto& rr : runlist) {
1580  //check run
1581  if (!CheckRunfileUpToDate(data_type, rr, _or)) {
1582  cout << " *** run " << rr << " needs update ***" <<
1583  endl;
1584  cout << "\t\tREPOSITORY: " << fRepository->
1585  GetName() << "\tDATE: " << GetRunfileDate(data_type, rr).value().AsString() << endl;
1586  cout << "\t\tREPOSITORY: " << other_repos << "\tDATE: " << _or->
1587  GetDataSetManager()->GetDataSet(GetName())->
1588  GetRunfileDate(data_type, rr).value().AsString() << endl;
1589  need_update++;
1590  }
1591  }
1592  if (!need_update) {
1593  cout << " *** All runfiles are up to date for data type " <<
1594  data_type << endl;
1595  }
1596 }
1597 
1598 
1599 
1600 
1604 
1605 run_index_list KVDataSet::GetUpdatableRuns(const Char_t* data_type,
1606  const Char_t* other_repos)
1607 {
1608  //Returns list of all runs of type "data_type" which may be updated
1609  //from the repository named "other_repos". See CheckUpToDate().
1610 
1611  run_index_list updates;
1612  if (!fRepository) return updates;
1613 
1614  KVDataRepository* _or =
1615  gDataRepositoryManager->GetRepository(other_repos);
1616  if (!_or) {
1617  Error("CheckUpToDate",
1618  "No data repository known with this name : %s", other_repos);
1619  return updates;
1620  }
1621  auto runlist = GetAvailableRunsFile(data_type)->GetRunList();
1622  for (auto& rr : runlist) {
1623  //check run
1624  if (!CheckRunfileUpToDate(data_type, rr, _or)) {
1625  //run is out of date
1626  updates.Add(rr);
1627  }
1628  }
1629  return updates;
1630 }
1631 
1632 
1633 
1634 
1639 
1640 run_index_list KVDataSet::GetRunList(const Char_t* data_type,
1641  const KVDBSystem* system) const
1642 {
1643  //Returns list of all files available for given "data_type"
1644  //
1645  //If a pointer to a reaction system is given, only files for the given system will be included in the list.
1646 
1647  if (!fRepository || !HasDataType(data_type)) {
1648  Error("GetRunList",
1649  "No data of type %s available. Runlist will be empty.",
1650  data_type);
1651  return {};
1652  }
1653 
1654  return GetAvailableRunsFile(data_type)->GetRunList(system);
1655 }
1656 
1657 
1658 
1659 
1690 
1692 {
1693  //This method returns a pointer to the available analysis task whose description (title) contains
1694  //all of the whitespace-separated keywords (which may be regular expressions)
1695  //given in the string "keywords". The comparison is case-insensitive.
1696  //
1697  //WARNING: this method can only be used to access analysis tasks that are
1698  //available for this dataset, i.e. for which the corresponding prerequisite data type
1699  //is available in the repository.
1700  //For unavailable data/tasks, use GetAnalysisTaskAny(const Char_t*).
1701  //
1702  //EXAMPLES
1703  //Let us suppose that the current dataset has the following list of tasks:
1704  //~~~
1705  // root [2] gDataSet->Print("tasks")
1706  // 1. Event reconstruction from raw data (raw->recon)
1707  // 2. Analysis of raw data
1708  // 3. Identification of reconstructed events (recon->ident)
1709  // 4. Analysis of reconstructed events (recon)
1710  // 5. Analysis of partially identified & calibrated reconstructed events (ident)
1711  // 6. Analysis of fully calibrated physical data (root)
1712  //~~~
1713  //Then the following will occur:
1714  //~~~
1715  // root [14] gDataSet->GetAnalysisTask("raw->recon")->Print()
1716  // KVDataAnalysisTask : Event reconstruction from raw data (raw->recon)
1717  //
1718  // root [10] gDataSet->GetAnalysisTask("analysis root")->Print()
1719  // KVDataAnalysisTask : Analysis of fully calibrated physical data (root)
1720  //~~~
1721 
1722  //case-insensitive search for matches in list based on 'title' attribute
1723  return (KVDataAnalysisTask*)fTasks.FindObjectAny("title", keywords, kTRUE, kFALSE);
1724 }
1725 
1726 
1727 
1728 
1730 
1732 {
1733  if (!dat->WithUserClass()) {
1734  Error("MakeAnalysisClass",
1735  "No user class required for analysis task \"%s\"", dat->GetTitle());
1736  return false;
1737  }
1738 
1739  //all analysis base classes must define a static Make(const Char_t * classname)
1740  //which generates the skeleton class files.
1741 
1742  TClass* cl = nullptr;
1743  //has the user base class for the task been compiled and loaded ?
1745  else
1746  return false;
1747 
1748  //set up call to static Make method
1749  unique_ptr<TMethodCall> methcall(new TMethodCall(cl, "Make", Form("\"%s\"", classname)));
1750 
1751  if (!methcall->IsValid()) {
1752  Error("MakeAnalysisClass", "static Make(const Char_t*) method for class %s is not valid",
1753  cl->GetName());
1754  return false;
1755  }
1756 
1757  //generate skeleton class
1758  methcall->Execute();
1759 
1760  return true;
1761 }
1762 
1763 
1764 
1777 
1778 bool KVDataSet::MakeAnalysisClass(const Char_t* task, const Char_t* classname)
1779 {
1780  //Create a skeleton analysis class to be used for analysis of the data belonging to this dataset.
1781  //
1782  // task = keywords contained in title of analysis task (see GetAnalysisTask(const Char_t*))
1783  // (you do not need to include 'analysis', it is added automatically)
1784  // classname = name of new analysis class
1785  //
1786  //Example:
1787  // MakeAnalysisClass("raw", "MyRawDataAnalysis")
1788  // --> make skeleton raw data analysis class in files MyRawDataAnalysis.cpp & MyRawDataAnalysis.h
1789  // MakeAnalysisClass("fully calibrated", "MyDataAnalysis")
1790  // --> make skeleton data analysis class in files MyDataAnalysis.cpp & MyDataAnalysis.h
1791 
1792  KVString _task = task;
1793  if (!_task.Contains("nalysis")) _task += " analysis";
1794  //We want to be able to write analysis classes even when we don't have any data
1795  //to analyse. Therefore we use GetAnalysisTaskAny.
1796  auto dat = GetAnalysisTaskAny(_task.Data());
1797  if (!dat) {
1798  Error("MakeAnalysisClass",
1799  "called for unknown or unavailable analysis task : %s", _task.Data());
1800  return false;
1801  }
1802  return make_analysis_class(dat, classname);
1803 }
1804 
1805 
1806 
1813 
1814 bool KVDataSet::MakeAnalysisClass(int task, const Char_t* classname)
1815 {
1816  //Create a skeleton analysis class to be used for analysis of the data belonging to this dataset.
1817  //
1818  // task = index of analysis task as shown in KVDataSet::Print("tasks")
1819  // classname = name of new analysis class
1820  //
1821 
1822  auto dat = GetAnalysisTask(task);
1823  if (!dat) {
1824  Error("MakeAnalysisClass",
1825  "called for unknown or unavailable analysis task index : %d", task);
1826  return false;
1827  }
1828  return make_analysis_class(dat, classname);
1829 }
1830 
1831 
1832 
1833 
1837 
1838 Bool_t KVDataSet::OpenDataSetFile(const Char_t* filename, ifstream& file)
1839 {
1840  // Look for (and open for reading, if found) the named file in the directory which
1841  // contains the files for this dataset (given by GetDataSetDir())
1842 
1843  return OpenDataSetFile(GetName(), filename, file);
1844 }
1845 
1846 
1847 
1851 
1852 Bool_t KVDataSet::OpenDataSetFile(const TString& dataset, const Char_t* filename, ifstream& file)
1853 {
1854  // Static method to look for (and open for reading, if found) the named file in the directory which
1855  // contains the files for the dataset
1856 
1857  TString datasetdir = KVBase::GetDataSetEnv(dataset, "DataSet.Directory", dataset);
1858  if (gSystem->IsAbsoluteFileName(datasetdir)) {
1859  // dataset directory is outside of standard KV installation directories
1860  // use absolute path to search for file
1861  TString abspath;
1862  abspath.Form("%s/%s", datasetdir.Data(), filename);
1863  return SearchAndOpenKVFile(abspath, file);
1864  }
1865  // dataset directory is a subdirectory of GetDATADIRFilePath()
1866  return SearchAndOpenKVFile(filename, file, datasetdir);
1867 }
1868 
1869 
1870 
1874 
1876 {
1877  // Find a file in the dataset directory (given by GetDataSetDir())
1878  // Returns full path to file if found, empty string if not
1879 
1880  return GetFullPathToDataSetFile(GetName(), filename);
1881 }
1882 
1883 
1884 
1888 
1890 {
1891  // Static method to find a file in the dataset directory (given by GetDataSetDir())
1892  // Returns full path to file if found, empty string if not
1893 
1894  TString fullpath;
1895  TString datasetdir = KVBase::GetDataSetEnv(dataset, "DataSet.Directory", dataset);
1896  if (!SearchKVFile(filename, fullpath, datasetdir)) {
1897  ::Warning("KVDataSet::GetFullPathToDataSetFile", "File %s not found in dataset subdirectory %s", filename, datasetdir.Data());
1898  fullpath = "";
1899  }
1900  return fullpath;
1901 }
1902 
1903 
1904 
1908 
1909 Bool_t KVDataSet::FindDataSetFile(const TString& dataset, const Char_t* filename)
1910 {
1911  // Static method to find a file in the dataset directory (given by GetDataSetDir())
1912  // Returns kTRUE if found, kFALSE if not
1913 
1914  TString fullpath;
1915  TString datasetdir = KVBase::GetDataSetEnv(dataset, "DataSet.Directory", dataset);
1916  return SearchKVFile(filename, fullpath, datasetdir);
1917 }
1918 
1919 
1920 
1924 
1926 {
1927  // Find a file in the dataset directory (given by GetDataSetDir())
1928  // Returns kTRUE if found, kFALSE if not
1929 
1930  return FindDataSetFile(GetName(), filename);
1931 }
1932 
1933 
1934 
1935 
1944 
1946 {
1947  //This method returns the analysis task whose description (title) contains
1948  //all of the whitespace-separated keywords (which may be regular expressions)
1949  //given in the string "keywords". The comparison is case-insensitive.
1950  //The analysis task does not need to be "available", i.e. the associated prerequisite
1951  //data type does not have to be present in the repository (see GetAnalysisTask).
1952  //
1953  // If no task is found, returns nullptr
1954 
1955  KVDataAnalysisTask* tsk = (KVDataAnalysisTask*)fAllTasks.FindObjectAny("title", keywords, kTRUE, kFALSE);
1956  if (!tsk) {
1957  Error("GetAnalysisTaskAny", "No task found with the following keywords in its title : %s", keywords);
1958  }
1959  return tsk;
1960 }
1961 
1962 
1963 
1964 
1970 
1972 {
1973  // Returns kTRUE if database needs to be regenerated from source files,
1974  // i.e. if source files are more recent than DataBase.root
1975  // In case no directory exists for dataset (dataset added 'on the fly')
1976  // we create the directory and fill it with dummy files (Makefile, Runlist.csv, Systems.dat)
1977 
1978  TString pwd = gSystem->pwd();
1979 
1980  TString path = "";
1981  if (!SearchKVFile(GetDataSetDir(), path)) {
1982  // dataset directory doesn't exist - create it
1983  Info("DataBaseNeedsUpdate", "%s: Creating new dataset directory %s",
1984  GetName(), GetDataSetDir());
1985  if (gSystem->mkdir(GetDataSetDir())) {
1986  // problem creating directory
1987  Error("DataBaseNeedsUpdate",
1988  "%s: Dataset directory %s does not exist and cannot be created ?",
1989  GetName(), GetDataSetDir());
1990  return kFALSE;
1991  }
1992  // create dummy files
1993  SearchKVFile(GetDataSetDir(), path); // get full path
1994  path += "/";
1995  TString filename = path + "Makefile";
1996  ofstream of1(filename.Data());
1997  of1 << "$(KV_WORK_DIR)/db/" << GetName() << "/DataBase.root : Runlist.csv Systems.dat" << endl;
1998  of1 << "\t@echo Database needs update" << endl;
1999  of1.close();
2000  filename = path + "Runlist.csv";
2001  ofstream of2(filename.Data());
2002  of2 << "# Automatically generated dummy Runlist.csv file" << endl;
2003  of2.close();
2004  filename = path + "Systems.dat";
2005  ofstream of3(filename.Data());
2006  of3 << "# Automatically generated dummy Systems.dat file" << endl;
2007  of3.close();
2008  }
2009  gSystem->cd(GetDataSetDir());
2010  TString cmd = "make -q";
2011  Int_t ret = gSystem->Exec(cmd.Data());
2012  gSystem->cd(pwd.Data());
2013  return (ret != 0);
2014 }
2015 
2016 
2017 
2034 
2036 {
2037  // Returns name of output repository for given task.
2038  // By default it is the name of the repository associated with this dataset,
2039  // but can be changed by the following environment variables:
2040  //
2041  // [repository].DefaultOutputRepository: [other repository]
2042  // - this means that all tasks carried out on data in [repository]
2043  // will have their output files placed in [other repository]
2044  //
2045  // [taskname].DataAnalysisTask.OutputRepository: [other repository]
2046  // - this means that for [taskname], any output files will
2047  // be placed in [other repository]
2048  //
2049  // [dataset].[taskname].DataAnalysisTask.OutputRepository: [other repository]
2050  // - this means that for given [dataset] & [taskname],
2051  // any output files will be placed in [other repository]
2052 
2053  if (gEnv->Defined(Form("%s.DataRepository.DefaultOutputRepository", GetRepository()->GetName())))
2054  return TString(gEnv->GetValue(Form("%s.DataRepository.DefaultOutputRepository", GetRepository()->GetName()), ""));
2055  TString orep = GetDataSetEnv(Form("%s.DataAnalysisTask.OutputRepository", taskname), GetRepository()->GetName());
2056  return orep;
2057 }
2058 
2059 
2060 
2061 
2064 
2065 void KVDataSet::CopyRunfilesFromRepository(const Char_t* type, const run_index_list& runs, const Char_t* destdir)
2066 {
2067  // Copies the runfiles of given "type" into the local directory "destdir".
2068 
2069  KVDataRepository* repo = GetRepository();
2070  for (auto& run : runs) {
2071  TString filename = GetRunfileName(type, run);
2072  TString destpath;
2073  AssignAndDelete(destpath, gSystem->ConcatFileName(destdir, filename));
2074  repo->CopyFileFromRepository(this, type, filename, destpath);
2075  }
2076 }
2077 
2078 
2079 
2080 
2084 
2085 void KVDataSet::CopyRunfilesToRepository(const Char_t* type, const run_index_list& runs, const Char_t* destrepo)
2086 {
2087  // Copies the runfiles of given "type" from the data repository associated
2088  // with this dataset into the local repository "destrepo".
2089 
2090  KVDataRepository* repo = GetRepository();
2091  KVDataRepository* dest_repo = gDataRepositoryManager->GetRepository(destrepo);
2092 
2093  if (!dest_repo) {
2094  Error("CopyRunfilesToRepository", "Unknown destination repository : %s", destrepo);
2095  gDataRepositoryManager->Print();
2096  return;
2097  }
2098 
2099  KVDataSet* dest_ds = dest_repo->GetDataSetManager()->GetDataSet(GetName());
2100  dest_repo->CreateAllNeededSubdirectories(dest_ds, type);
2101  for (auto& run : runs) {
2102  TString filename = GetRunfileName(type, run);
2103  TString destpath = dest_repo->GetFullPathToTransferFile(dest_ds, type, filename);
2104  repo->CopyFileFromRepository(this, type, filename, destpath);
2105  }
2106 }
2107 
2108 
int Int_t
ROOT::R::TRInterface & r
#define f(i)
bool Bool_t
char Char_t
constexpr Bool_t kFALSE
constexpr Bool_t kTRUE
const char Option_t
#define gDirectory
R__EXTERN TEnv * gEnv
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
char * Form(const char *fmt,...)
void AssignAndDelete(TString &target, char *tobedeleted)
R__EXTERN TSystem * gSystem
Handles lists of available runs for different datasets and types of data.
virtual void Remove(const run_index_t &run, const KVString &filename="")
Remove from the db ALL entries corresponding to the given run/index (and filename if given)
std::forward_list< run_index_t > CheckMultiRunfiles()
Returns a list with all run/indexes for which more than one file is in the available runs db.
void GetRunInfos(const run_index_t &run, KVList *dates, KVList *names)
virtual void Add(const run_index_t &run, const KVString &filename)
static ValType GetDataSetEnv(const KVString &dataset, const KVString &type, const ValType &defval)
Definition: KVBase.h:304
Database class used to store information on different colliding systems studied during an experiment....
Definition: KVDBSystem.h:52
Bool_t IsCollision() const
retourne kTRUE, si le systeme est une collision ie projectile+cible
Definition: KVDBSystem.cpp:105
Define and manage data analysis tasks.
virtual void SetDataAnalyser(const Char_t *d)
virtual Bool_t WithUserClass() const
virtual void SetPrereq(const Char_t *p)
virtual void SetStatusUpdateInterval(Long64_t n)
virtual const Char_t * GetUserBaseClass() const
virtual Bool_t CheckUserBaseClassIsLoaded() const
void SetExtraAClicIncludes(const KVString &list)
virtual const Char_t * GetPrereq() const
virtual void SetUserBaseClass(const Char_t *d)
KVDataRepository * GetRepository(const Char_t *name) const
void Print(Option_t *opt="") const
Base class for managing repositories of experimental data.
virtual KVDataSetManager * GetDataSetManager() const
Return pointer to data set manager for this repository.
virtual void CopyFileFromRepository(const KVDataSet *dataset, const Char_t *datatype, const Char_t *filename, const Char_t *destination)
void CreateAllNeededSubdirectories(const KVDataSet *DataSet, const Char_t *DataType)
virtual const Char_t * GetFullPathToTransferFile(const KVDataSet *dataset, const Char_t *datatype, const Char_t *runfile)
KVDataSet * GetDataSet(Int_t) const
Return pointer to DataSet using index in list of all datasets, index>=0.
Manage an experimental dataset corresponding to a given experiment or campaign.
Definition: KVDataSet.h:146
std::optional< TDatime > GetRunfileDate(const KVString &type, const run_index_t &run)
Definition: KVDataSet.cpp:838
TString GetFullPathToRunfile(const KVString &type, const run_index_t &run) const
Definition: KVDataSet.cpp:794
TString GetFullPathToDB() const
Definition: KVDataSet.cpp:153
void ls(Option_t *opt="") const override
Print dataset information.
Definition: KVDataSet.cpp:377
TString GetOutputRepository(const Char_t *taskname) const
Definition: KVDataSet.cpp:2035
virtual void OpenDBFile(const Char_t *full_path_to_dbfile) const
Open the database from a file on disk.
Definition: KVDataSet.cpp:127
run_index_list GetRunList_VersionSelection(const Char_t *type, const Char_t *version, KVDBSystem *sys=0)
Definition: KVDataSet.cpp:1246
TString GetDBName() const
Definition: KVDataSet.cpp:112
void CheckMultiRunfiles(const Char_t *data_type)
Definition: KVDataSet.cpp:1388
Bool_t CheckRunfileAvailable(const KVString &type, const run_index_t &run)
Definition: KVDataSet.cpp:934
virtual const Char_t * GetBaseFileName(const Char_t *type, const run_index_t &run) const
Definition: KVDataSet.cpp:967
void CheckUpToDate(const Char_t *data_type, const Char_t *other_repos)
Definition: KVDataSet.cpp:1562
run_index_list GetRunList(const Char_t *data_type, const KVDBSystem *sys=0) const
Definition: KVDataSet.cpp:1640
void CopyRunfilesFromRepository(const Char_t *type, const run_index_list &runs, const Char_t *destdir)
Copies the runfiles of given "type" into the local directory "destdir".
Definition: KVDataSet.cpp:2065
KVDataRepository * GetRepository() const
Get pointer to data repository in which dataset is stored.
Definition: KVDataSet.cpp:1374
const Char_t * GetDataSetDir() const
Definition: KVDataSet.cpp:670
KVAvailableRunsFile * GetAvailableRunsFile(const Char_t *type) const
Definition: KVDataSet.cpp:51
void CommitRunfile(const KVString &type, const run_index_t &run, TFile *file)
Definition: KVDataSet.cpp:1287
virtual void AddAvailableDataType(const Char_t *)
Definition: KVDataSet.cpp:490
bool make_analysis_class(const KVDataAnalysisTask *, const Char_t *classname)
Definition: KVDataSet.cpp:1731
KVDataAnalysisTask * GetAnalysisTask(Int_t) const
Definition: KVDataSet.cpp:556
virtual void SaveDataBase() const
Definition: KVDataSet.cpp:203
virtual std::unique_ptr< TList > GetListOfAvailableSystems(const Char_t *datatype, KVDBSystem *systol=0)
Definition: KVDataSet.cpp:573
virtual KVDataAnalysisTask * GetAnalysisTaskAny(const Char_t *keywords) const
Definition: KVDataSet.cpp:1945
run_index_list GetRunList_DateSelection(const Char_t *type, TDatime *min=0, TDatime *max=0)
Definition: KVDataSet.cpp:1130
virtual void SetAnalysisTasks(const KVSeqCollection *)
Definition: KVDataSet.cpp:507
TFile * NewRunfile(const KVString &type, const run_index_t &run)
Definition: KVDataSet.cpp:1027
Bool_t CheckRunfileUpToDate(const KVString &data_type, const run_index_t &run, KVDataRepository *other_repos)
Definition: KVDataSet.cpp:1503
virtual void CheckAvailable()
Definition: KVDataSet.cpp:443
void DeleteRunfile(const KVString &type, const run_index_t &run, Bool_t confirm=kTRUE)
Definition: KVDataSet.cpp:1052
virtual Int_t GetNtasks() const
Definition: KVDataSet.cpp:541
run_index_list GetRunList_StageSelection(const Char_t *other_type, const Char_t *base_type, KVDBSystem *sys=0, Bool_t OnlyCol=kFALSE)
Definition: KVDataSet.cpp:1192
TString GetFullPathToDataSetFile(const Char_t *filename)
Definition: KVDataSet.cpp:1875
void DeleteRunfiles(const Char_t *type, const run_index_list &lrun={}, Bool_t confirm=kTRUE)
Definition: KVDataSet.cpp:1099
TObject * open_runfile(const KVString &type, const run_index_t &run)
Definition: KVDataSet.cpp:742
void CopyRunfilesToRepository(const Char_t *type, const run_index_list &runs, const Char_t *destrepo)
Definition: KVDataSet.cpp:2085
KVString GetRunfileName(const KVString &type, const run_index_t &run) const
Definition: KVDataSet.cpp:814
KVExpDB * GetDataBase(Option_t *opt="") const
Definition: KVDataSet.cpp:285
virtual Bool_t DataBaseNeedsUpdate() const
Definition: KVDataSet.cpp:1971
Bool_t OpenDataSetFile(const Char_t *filename, std::ifstream &file)
void SetDataSetSpecificTaskParameters(KVDataAnalysisTask *) const
Check configuration variables to see if the task parameters have been "tweaked" for the dataset.
Definition: KVDataSet.cpp:178
void cd() const
Definition: KVDataSet.cpp:692
virtual void WriteDBFile(const Char_t *full_path_to_dbfile) const
Write the database to disk.
Definition: KVDataSet.cpp:255
void Print(Option_t *opt="") const override
Definition: KVDataSet.cpp:400
void SetName(const char *name) override
Definition: KVDataSet.cpp:632
virtual Bool_t CheckUserCanAccess()
Definition: KVDataSet.cpp:1331
void UpdateAvailableRuns(const KVString &type)
Definition: KVDataSet.cpp:1005
run_index_list GetUpdatableRuns(const Char_t *data_type, const Char_t *other_repos)
Definition: KVDataSet.cpp:1605
void SetRepository(KVDataRepository *)
Set pointer to data repository in which dataset is stored.
Definition: KVDataSet.cpp:1362
bool MakeAnalysisClass(const Char_t *task, const Char_t *classname)
Definition: KVDataSet.cpp:1778
KVDataSet()
Default constructor.
Definition: KVDataSet.cpp:35
std::optional< run_index_t > GetRunNumberFromFileName(const TString &datatype, const TString &filename)
Definition: KVDataSet.cpp:887
void CleanMultiRunfiles(const Char_t *data_type, Bool_t confirm=kTRUE)
Definition: KVDataSet.cpp:1432
virtual void OpenDataBase(Option_t *opt="") const
Definition: KVDataSet.cpp:319
static Bool_t FindDataSetFile(const TString &dataset, const Char_t *filename)
Definition: KVDataSet.cpp:1909
TString GetDBFileName() const
Definition: KVDataSet.cpp:83
Extension of TDatime to handle various useful date formats.
Definition: KVDatime.h:33
Base class to describe database of an experiment ,,.
Definition: KVExpDB.h:61
virtual void cd()
Definition: KVExpDB.cpp:576
static KVExpDB * MakeDataBase(const Char_t *name, const Char_t *datasetdir)
Definition: KVExpDB.cpp:598
Extended TList class which owns its objects by default.
Definition: KVList.h:28
Description of an individual data file in an experimental dataset.
Definition: KVRunFile.h:19
const Char_t * GetVersion() const
Definition: KVRunFile.h:84
const KVDatime & GetFileWrittenDatime() const
Definition: KVRunFile.h:76
const run_index_t & GetRunIndex() const
Definition: KVRunFile.h:56
KaliVeda extensions to ROOT collection classes.
TObject * At(Int_t idx) const override
Extension of ROOT TString class which allows backwards compatibility with ROOT v3....
Definition: KVString.h:73
void Begin(TString delim) const
Definition: KVString.cpp:565
Bool_t End() const
Definition: KVString.cpp:634
KVString Next(Bool_t strip_whitespace=kFALSE) const
Definition: KVString.cpp:695
static TClass * GetClass(Bool_t load=kTRUE, Bool_t silent=kFALSE)
virtual Int_t GetEntries() const
const char * AsSQLString() const
virtual Bool_t cd()
virtual const char * GetValue(const char *name, const char *dflt) const
virtual void SetValue(const char *name, const char *value, EEnvLevel level=kEnvChange, const char *type=nullptr)
Bool_t Defined(const char *name) const
TObject * FindObject(const char *name) const override
virtual void SetTitle(const char *title="")
const char * GetName() const override
const char * GetTitle() const override
virtual void SetName(const char *name)
TString & String()
const char * Data() const
void ToUpper()
void Form(const char *fmt,...)
TString & Remove(EStripType s, char c)
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
TString & ReplaceAll(const char *s1, const char *s2)
Bool_t cd(const char *path)
const char * pwd()
virtual int Chmod(const char *file, UInt_t mode)
virtual const char * DirName(const char *pathname)
virtual char * ConcatFileName(const char *dir, const char *name)
virtual int mkdir(const char *name, Bool_t recursive=kFALSE)
virtual Int_t Exec(const char *shellcmd)
virtual Bool_t AccessPathName(const char *path, EAccessMode mode=kFileExists)
virtual const char * BaseName(const char *pathname)
virtual Bool_t IsAbsoluteFileName(const char *dir)
Specifies a runfile according to run number and file index ,.
Definition: run_index.h:31
int run() const
Definition: run_index.h:50
TString as_string() const
Definition: run_index.h:95
int index(int no_index=-1) const
Definition: run_index.h:55
bool has_index() const
Definition: run_index.h:59
const Int_t n
void Error(const char *location, const char *fmt,...)
void Info(const char *location, const char *fmt,...)
void Warning(const char *location, const char *fmt,...)
double min(double x, double y)
double max(double x, double y)
const char * String
v
TArc a
ClassImp(TPyArg)