9 #include "KVDataAnalyser.h"
10 #include "KVDataAnalysisTask.h"
11 #include "KVDataSetManager.h"
16 #include "KVBatchSystemManager.h"
48 fWorkDirInit = fWorkDirEnd = 0;
51 fProofMode = EProofMode::None;
55 fUseBaseClassSubmitTask =
kFALSE;
63 KVDataAnalyser::~KVDataAnalyser()
68 if (gDataAnalyser ==
this)gDataAnalyser =
nullptr;
83 fUserClassOptions =
"";
85 fBatchSystem =
nullptr;
88 fProofMode = EProofMode::None;
108 if (CheckTaskVariables()) {
111 fBatchSystem->SubmitTask(
this);
114 if (
BatchMode() && fBatchSystem && !fParent) fBatchSystem->Print(
"log");
115 if (!RunningInLaunchDirectory() && fParent) {
119 ScanWorkingDirectory(&fWorkDirInit);
121 if (!PreSubmitCheck())
return;
131 if (fUseBaseClassSubmitTask)
136 if (!RunningInLaunchDirectory() && fParent) {
140 ScanWorkingDirectory(&fWorkDirEnd);
143 CopyAnalysisResultsToLaunchDirectory();
145 if (
BatchMode() && fBatchSystem && !fParent) {
179 if (NeedToChooseWhatToDo())
181 else if (NeedToChooseWhatToAnalyse())
182 ChooseWhatToAnalyse();
205 if (
BatchMode()) ReadBatchEnvFile(
Form(
".%s", GetBatchName()));
209 if (fTask->WithUserClass() && fUserClass.name !=
ClassName()) {
211 if (fUserClass.name.IsNull()) {
216 if (!CheckIfUserClassIsValid(fUserClassAlternativeBaseClass)) {
217 cout <<
"============> Warning <=============" << endl;
218 cout << GetUserClassName() <<
" is not a valid " << fTask->GetUserBaseClass() << endl;
219 cout <<
"Analysis aborted." << endl;
220 cout <<
"====================================" << endl;
226 if (nbEventToRead < 0) {
227 ChooseNbEventToRead();
239 if (NeedToChooseWhatToDo()) {
243 if (NeedToChooseWhatToAnalyse()) {
296 fIncludes +=
id.
Data();
298 if (!curIncDir.
Contains(
id.Data())) {
299 cout <<
"Include path \"" <<
id.Data() <<
"\" added." << endl;
334 while (!slib.
End() && !loaded) {
337 Info(
"SetUserLibraries",
"%s already load",
id.Data());
344 Info(
"SetUserLibraries",
"Library \"%s\"added.",
id.Data());
347 fLibraries +=
id.Data();
363 SetNbEventToRead(-1);
364 while (nbEventToRead < 0) {
365 cout <<
"Give the number of events to read [<RET>=all]:" << endl;
378 Data() <<
"\" is not a number. Please retry." << endl;
414 fUserClass.name =
"";
415 while (!fUserClass.name.Length()) {
416 cout <<
"Give the name of the analysis class derived from " << fTask->GetUserBaseClass() <<
":" << endl;
417 fUserClass.name.ReadLine(cin);
435 fUserClass.imp_file, fUserClass.dec_file, fUserClass.dir_name);
476 TObject* o = GetInstanceOfUserClass(alternative_base_class);
526 fBatchSystem =
nullptr;
569 if (!fTask->CheckUserBaseClassIsLoaded())
return 0x0;
572 TPluginHandler* ph =
gROOT->GetPluginManager()->FindHandler(fTask->GetUserBaseClass(), fUserClass.name);
580 if (DoUserClassFilesExist()) {
583 cmd.
Form(
".L %s%s", fUserClass.full_path_imp().Data(), GetACliCMode());
586 cl =
gROOT->GetClass(fUserClass.name);
590 Info(
"GetInstanceOfUserClass",
"Class %s is unknown and no source files available",
591 fUserClass.name.Data());
596 Info(
"GetInstanceOfUserClass",
"Compilation of class %s failed. Correct the mistakes and try again",
597 fUserClass.name.Data());
600 if (!cl->GetBaseClass(fTask->GetUserBaseClass())) {
602 if (alternative_base_class ==
"") {
603 Info(
"GetInstanceOfUserClass",
"Class %s does not inherit from correct base class (%s), or compilation of class %s failed. Correct the mistakes and try again",
604 fUserClass.name.Data(), fTask->GetUserBaseClass(), fUserClass.name.Data());
609 bool got_good_base =
false;
611 alternative_base_class.
Begin(
",");
612 while (!alternative_base_class.
End()) {
613 good_base = alternative_base_class.
Next(
kTRUE);
614 if (cl->GetBaseClass(good_base)) {
615 got_good_base =
true;
620 Info(
"GetInstanceOfUserClass",
"Class %s inherits from alternative base class %s: OK!",
621 fUserClass.name.Data(), good_base.
Data());
624 Info(
"GetInstanceOfUserClass",
"Class %s does not inherit from task-defined base class (%s) or any provided alternative base classes (%s), or compilation of class %s failed. Correct the mistakes and try again",
625 fUserClass.name.Data(), fTask->GetUserBaseClass(), alternative_base_class.
Data(), fUserClass.name.Data());
634 Info(
"GetInstanceOfUserClass",
"Found plugin handler for class %s",
635 fUserClass.name.Data());
639 Info(
"GetInstanceOfUserClass",
"KVBase::LoadPlugin failed for %s", fUserClass.name.Data());
645 if (obj->
InheritsFrom(fTask->GetUserBaseClass()))
return obj;
646 Info(
"GetInstanceOfUserClass",
"%s does not inherit from %s", fUserClass.name.Data(), fTask->GetUserBaseClass());
650 Info(
"GetInstanceOfUserClass",
"constructor not OK for %s", fUserClass.name.Data());
673 fUserClassIsOK = CheckIfUserClassIsValid();
676 fUserClassIsOK =
kTRUE;
696 OpenBatchInfoFile(
Form(
".%s", jobname.
Data()));
698 fBatchEnv->SetValue(
"BatchSystem", fBatchSystem->GetName());
699 fBatchSystem->WriteBatchEnvFile(GetBatchInfoFile());
701 fBatchEnv->SetValue(
"AnalysisTask", fTask->GetType());
703 if (fTask->WithUserClass()) {
704 fBatchEnv->SetValue(
"UserClass", GetUserClass().
name);
705 if (!GetUserClass().with_files()) {
706 if (!DoUserClassFilesExist()) {
707 Warning(
"WriteBatchEnvFile",
"Source files for user class %s do not exist. Job will not work.",
708 GetUserClass().
name.Data());
711 if (!GetUserClass().dir_name.IsNull()) {
714 fBatchEnv->SetValue(
"UserClassDir", GetUserClass().dir_name);
716 fBatchEnv->SetValue(
"UserClassOptions", fUserClassOptions);
717 fBatchEnv->SetValue(
"UserClassImp", GetUserClass().imp_file);
718 fBatchEnv->SetValue(
"UserClassDec", GetUserClass().dec_file);
722 if (fUserClassOptions !=
"") fBatchEnv->SetValue(
"UserClassOptions", fUserClassOptions);
724 fBatchEnv->SetValue(
"NbToRead", (
Double_t)nbEventToRead);
726 if (fIncludes.Length()) {
727 fBatchEnv->SetValue(
"UserIncludes", fIncludes.Data());
729 if (fLibraries.Length()) {
730 fBatchEnv->SetValue(
"UserLibraries", fLibraries.Data());
733 if (save) fBatchEnv->SaveLevel(
kEnvUser);
754 if (!GetAnalysisTask()) {
758 KVString val = fBatchEnv->GetValue(
"AnalysisTask",
"");
759 SetAnalysisTask(
nullptr);
761 if (!gDataSetManager) {
763 gDataSetManager->
Init();
768 Error(
"ReadBatchEnvFile",
"Name of analysis task not given");
771 if (!GetAnalysisTask()) {
772 Error(
"ReadBatchEnvFile",
"Analysis task \"%s\"not found for dataset %s",
778 nbEventToRead = (
Long64_t)fBatchEnv->GetValue(
"NbToRead", -1);
779 SetUserIncludes(fBatchEnv->GetValue(
"UserIncludes",
""));
780 SetUserLibraries(fBatchEnv->GetValue(
"UserLibraries",
""));
783 if (strcmp(fBatchEnv->GetValue(
"BatchSystem",
""),
"")) {
784 fBatchSystem = gBatchSystemManager->
GetBatchSystem(fBatchEnv->GetValue(
"BatchSystem",
""));
787 fBatchSystem->SetAnalyser(
this);
791 if (fTask->WithUserClass()) {
792 fUserClass.name = fBatchEnv->GetValue(
"UserClass",
"");
793 if (fUserClass.name.IsNull()) {
794 Error(
"ReadBatchEnvFile",
"Name of user class not given");
798 fUserClassOptions = fBatchEnv->GetValue(
"UserClassOptions",
"");
799 fUserClass.dir_name = fBatchEnv->GetValue(
"UserClassDir",
"");
800 fUserClass.imp_file = fBatchEnv->GetValue(
"UserClassImp",
"");
801 if (fUserClass.imp_file.IsNull()) {
802 Error(
"ReadBatchEnvFile",
"Name of user class implementation file not given");
805 fUserClass.dec_file = fBatchEnv->GetValue(
"UserClassDec",
"");
806 if (fUserClass.dec_file.IsNull()) {
807 Error(
"ReadBatchEnvFile",
"Name of user class header file not given");
810 fUserClassAlternativeBaseClass = fBatchEnv->GetValue(
"UserClassAlternativeBaseClass",
"");
814 if (!RunningInLaunchDirectory()) {
819 Info(
"ReadBatchEnvFile",
"Copying %s to %s", fUserClass.full_path_imp().Data(), path_trg.
Data());
823 Info(
"ReadBatchEnvFile",
"Copying %s to %s", fUserClass.full_path_dec().Data(), path_trg.
Data());
826 fUserClass.dir_name =
".";
831 fUserClassOptions = fBatchEnv->GetValue(
"UserClassOptions",
"");
886 KVString task_data_analyser = fTask->GetDataAnalyser();
887 Info(
"SubmitTask",
"fTask->GetDataAnalyser()=%s", task_data_analyser.
Data());
888 unique_ptr<KVDataAnalyser> the_analyser;
889 if (task_data_analyser ==
"UserClass") {
894 the_analyser.reset(GetAnalyser(fTask->GetDataAnalyser()));
896 if (!the_analyser.get())
897 Fatal(
"SubmitTask",
"the_analyser is 0x0, go to crash");
899 set_up_analyser_for_task(the_analyser.get());
901 if (fTask->WithUserClass()) {
902 the_analyser->SetUserClass(GetUserClass(),
kFALSE);
903 the_analyser->SetUserClassOptions(fUserClassOptions);
905 else if (strcmp(fTask->GetUserBaseClass(),
""))
906 the_analyser->SetUserClass({fTask->GetUserBaseClass()},
kFALSE);
911 the_analyser->CheckTaskVariables();
912 if (!fChoseRunMode) ChooseRunningMode();
915 the_analyser->SetBatchName(GetBatchName());
916 the_analyser->SetBatchSystem(fBatchSystem);
917 the_analyser->SetProofMode(GetProofMode());
920 gDataAnalyser = the_analyser.get();
947 if (fUserClass.name.Length()) tmp.
ReplaceAll(
"$UserClass", fUserClass.name);
948 else if (fTask) tmp.
ReplaceAll(
"$UserClass", fTask->GetDataAnalyser());
959 static TString keywords =
"Will be expanded: $Date, $User, $UserClass";
1012 if (!
BatchMode() || !fBatchEnv)
return "";
1018 TString launchDir = GetLaunchDirectory();
1034 TEnv stats(GetBatchStatusFileName());
1035 stats.
SetValue(
"TotalEvents", totev);
1036 stats.
SetValue(
"EventsRead", evread);
1054 TString stats = GetBatchStatusFileName();
1070 return (!(nevents % fTask->GetStatusUpdateInterval()) && nevents);
1082 cout <<
" +++ " << nevents <<
" events processed +++ " << endl;
1085 cout <<
" ------------- Process infos -------------" << endl;
1086 printf(
" CpuSys = %f s. CpuUser = %f s. ResMem = %f MB VirtMem = %f MB\n",
1106 fChoseRunMode =
kTRUE;
1109 cout << endl <<
"Run in Interactive or Batch mode (I or B) ? : ";
1112 while (tmp !=
"i" && tmp !=
"I" && tmp !=
"b" && tmp !=
"B");
1119 cout << endl <<
"Choose the batch system to use : " << endl;
1120 gBatchSystemManager->
Print();
1122 cout <<
"(enter a number) : " << endl;
1127 fBatchSystem->
Clear();
1141 if (*ls)
delete (*ls);
1160 if (!fWorkDirInit || !fWorkDirEnd)
return;
1162 TIter next_new_file(fWorkDirEnd);
1164 while ((
file = next_new_file())) {
1165 if (!fWorkDirInit->FindObject(
file->GetName())) {
1175 Info(
"CopyAnalysisResultsToLaunchDirectory",
"Copying analysis results file :\n%s ---> %s",
1179 Info(
"CopyAnalysisResultsToLaunchDirectory",
"File copied correctly");
1182 Info(
"CopyAnalysisResultsToLaunchDirectory",
" **** ERROR copying file !!! ");
1203 tt->GetUserInfo()->Add(
new TEnv());
1204 TEnv* kvenv = (
TEnv*)
tt->GetUserInfo()->FindObject(
"TEnv");
1220 #ifdef WITH_GIT_INFOS
1269 ::Warning(
"KVDataAnalyser::RunAnalyser",
"No plugin %s found for KVDataAnalyser",
1314 jdl->
SetValue(
"AnalysisTask", fTask->GetType());
1315 jdl->
SetValue(
"PROOFMode", GetProofMode());
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t format
char * Form(const char *fmt,...)
void AssignAndDelete(TString &target, char *tobedeleted)
R__EXTERN TSystem * gSystem
static const Char_t * GetKVBuildDate()
Returns KaliVeda build date.
static const Char_t * gitCommit()
Returns last git commit of sources.
static const Char_t * GetKVBuildUser()
Returns username of person who performed build.
static const Char_t * gitBranch()
Returns git branch of sources.
static Bool_t FindClassSourceFiles(const KVString &class_name, KVString &imp_file, KVString &dec_file, const KVString &dir_name=".")
static const Char_t * GetKVSourceDir()
Returns top-level directory of source tree used for build.
static void PrintSplashScreen()
Prints welcome message and infos on version etc.
static TPluginHandler * LoadPlugin(const Char_t *base, const Char_t *uri="0")
static const Char_t * GetKVVersion()
Returns KaliVeda version string.
Handles list of all available batch systems for processing non-interactive data analysis tasks.
void Print(Option_t *opt="") const
KVBatchSystem * GetBatchSystem(const Char_t *name)
Get batch system by name.
void Clear(Option_t *opt="") override
virtual const Char_t * GetJobName() const
virtual void ReadBatchEnvFile(TEnv *)
Manager class which sets up and runs data analysis tasks.
const Char_t * GetACliCMode()
void DoStatusUpdate(Long64_t nevents) const
Print infos on events treated, disk usage, memory usage.
virtual TString ExpandAutoBatchName(const Char_t *format) const
void SetParent(KVDataAnalyser *da)
virtual void PostRunReset()
virtual void Run()
Check all task variables, then run analyser.
virtual void AddJobDescriptionList(TList *)
virtual void WriteBatchEnvFile(const TString &, Bool_t sav=kTRUE)
void ChooseNbEventToRead()
Ask user to set number of events to read.
Bool_t RunningInLaunchDirectory()
void SetUserLibraries(const Char_t *libs=0)
virtual Bool_t CheckIfUserClassIsValid(const KVString &alternative_base_class="")
TString GetPathToFileInLaunchDirectory(const TString &) const
void WriteBatchInfo(TTree *)
virtual void RunMenus()
Run data analyser in menu-driven mode.
virtual Bool_t CheckTaskVariables()
Bool_t DoUserClassFilesExist()
const Char_t * GetLaunchDirectory() const
void SetAnalysisTask(KVDataAnalysisTask *at)
virtual const Char_t * GetRecognisedAutoBatchNameKeywords() const
virtual void set_up_analyser_for_task(KVDataAnalyser *the_analyser)
virtual Bool_t CheckStatusUpdateInterval(Long64_t nevents) const
virtual Bool_t CheckWhatToAnalyseAndHow()
virtual void SubmitTask()
void SetUserClass(const UserClass &kvs, Bool_t check=kTRUE)
const Char_t * GetBatchStatusFileName() const
Returns full path to file used to store status of running batch jobs.
virtual void ChooseWhatToAnalyse()
TO IMPLEMENT ?
KVDataAnalyser()
Default constructor.
static Bool_t fCleanAbort
flag to force abort of processing
static KVDataAnalyser * GetAnalyser(const Char_t *plugin)
static Bool_t IsRunningBatchAnalysis()
static void RunAnalyser(const Char_t *plugin="")
void ScanWorkingDirectory(TList **)
void ChooseUserClass()
Choose the user's analysis class.
void CopyAnalysisResultsToLaunchDirectory()
virtual Bool_t ReadBatchEnvFile(const TString &)
void DeleteBatchStatusFile() const
Delete batch status file (and backup - '.bak') for batch job.
void SetUserIncludes(const Char_t *incDirs=0)
virtual void ChooseWhatToDo()
TO IMPLEMENT ?
void UpdateBatchStatusFile(Int_t totev, Int_t evread, TString disk) const
Update infos in batch status file.
KVBatchSystem * fBatchSystem
batch system to use for submission of job
TObject * GetInstanceOfUserClass(const KVString &alternative_base_class="")
void SetNbEventToRead(Long64_t nb=0)
Define and manage data analysis tasks.
virtual const Char_t * GetPrereq() const
Manage all datasets contained in a given data repository.
virtual Bool_t Init(KVDataRepository *=0)
KVDataAnalysisTask * GetAnalysisTaskAny(const Char_t *keywords) const
Handles lists of named parameters with different types, a list of KVNamedParameter objects.
void SetValue(const Char_t *name, value_type value)
Extension of ROOT TString class which allows backwards compatibility with ROOT v3....
void Begin(TString delim) const
KVString Next(Bool_t strip_whitespace=kFALSE) const
virtual Int_t GetEntries() const
virtual void SetOwner(Bool_t enable=kTRUE)
const char * AsSQLString() const
THashList * GetTable() const
virtual const char * GetValue(const char *name, const char *dflt) const
virtual void SetValue(const char *name, const char *value, EEnvLevel level=kEnvChange, const char *type=nullptr)
virtual void SaveLevel(EEnvLevel level)
TObject * At(Int_t idx) const override
const char * GetName() const override
const TString & GetString() const
virtual const char * GetName() const
virtual Bool_t InheritsFrom(const char *classname) const
Longptr_t ExecPlugin(int nargs)
std::istream & ReadToDelim(std::istream &str, char delim='\n')
Bool_t EndsWith(const char *pat, ECaseCompare cmp=kExact) const
const char * Data() const
TObjArray * Tokenize(const TString &delim) const
Bool_t BeginsWith(const char *s, ECaseCompare cmp=kExact) const
void Form(const char *fmt,...)
TString & Remove(EStripType s, char c)
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
TString & ReplaceAll(const char *s1, const char *s2)
std::istream & ReadLine(std::istream &str, Bool_t skipWhite=kTRUE)
virtual TList * GetListOfFiles() const
virtual const char * GetBuildNode() const
virtual const char * GetBuildCompilerVersion() const
virtual const char * GetBuildDir() const
virtual void AddIncludePath(const char *includePath)
virtual int CopyFile(const char *from, const char *to, Bool_t overwrite=kFALSE)
virtual const char * Getenv(const char *env)
virtual const char * GetIncludePath()
virtual char * ConcatFileName(const char *dir, const char *name)
virtual int Load(const char *module, const char *entry="", Bool_t system=kFALSE)
virtual UserGroup_t * GetUserInfo(const char *user=nullptr)
virtual int GetProcInfo(ProcInfo_t *info) const
virtual const char * HostName()
virtual char * ExpandPathName(const char *path)
virtual const char * WorkingDirectory()
virtual const char * GetBuildCompiler() const
virtual const char * GetLibraries(const char *regexp="", const char *option="", Bool_t isRegexp=kTRUE)
virtual const char * GetBuildArch() const
virtual int Unlink(const char *name)
RooCmdArg ClassName(const char *name)
RooCmdArg BatchMode(std::string const &batchMode)
KVString get_full_path(const KVString &, const KVString &="") const