KVDataSetManager.cpp

/*
$Id: KVDataSetManager.cpp,v 1.17 2007/10/01 15:03:38 franklan Exp $
$Revision: 1.17 $
$Date: 2007/10/01 15:03:38 $
$Author: franklan $
*/
 
#include "KVBase.h"
#include "KVDataSetManager.h"
#include "KVDataRepositoryManager.h"
#include "KVDataSetRepository.h"
#include "KVString.h"
#include "TObjString.h"
#include "TObjArray.h"
#include "Riostream.h"
#include "KVBase.h"
#include "TPluginManager.h"
#include "TError.h"
 
using namespace std;
 
ClassImp(KVDataSetManager)
 
KVDataSetManager* gDataSetManager;
 
 
 
KVDataSetManager::KVDataSetManager()
   : fEnv(*gEnv)
{
   fNavailable = 0;
   fRepository = 0;
   fCacheAvailable = kFALSE;
   fMaxCacheTime = 0;
   fCacheFileName = "";
   fDataSets.SetOwner();
   fTasks.SetOwner();
}
 
 
 
 
Bool_t KVDataSetManager::Init(KVDataSetRepository* dr)
{
   //Initialisation of dataset manager for the repository 'dr'.
   //
   //If dr=0x0 (default) then all known datasets are 'available', otherwise
   //we check availability of datasets based on data present in repository.
   //
   //Initialise all possible data analysis tasks,
   //then set list of possible tasks for each available dataset.
   //
   //\returns kTRUE if all goes well.
 
   fRepository = dr;
 
   if (!ReadDataSetList())
      return kFALSE;
 
   //use caching for dataset availability ?
   if (dr) {
      fCacheAvailable = fEnv.GetValueOf(dr->GetName(), "DataRepository.CacheAvailable").Default(kFALSE);
      fMaxCacheTime = fEnv.GetValueOf(dr->GetName(), "DataRepository.MaxCacheTime").Default(0);
   }
   //name of cache file
   if (dr) fCacheFileName.Form("%s.available.datasets", dr->GetName());
 
   //check which datasets are available
   CheckAvailability();
 
   if (!ReadTaskList())
      return kFALSE;
 
   //set available data analysis tasks for available datasets
   if (GetNavailable()) {
      for (Int_t i = 1; i <= GetNavailable(); i++)
         GetAvailableDataSet(i)->SetAnalysisTasks(&fTasks);
   }
 
   // stand-alone dataset manager: make it the default
   if (!dr) gDataSetManager  = this;
 
   return kTRUE;
}
 
 
 
 
Bool_t KVDataSetManager::ReadDataSetList()
{
   //Initialise list of all known datasets from informations in .kvrootrc config files
   //(and user's .kvrootrc)
 
   KVString manip_list = fEnv.GetValueOf("DataSet").Default("");
 
   fDataSets.Clear();
 
   manip_list.Begin(" ");
 
   while (!manip_list.End()) {
 
      auto manip = manip_list.Next();
 
      KVDataSet* ds = NewDataSet();
      ds->SetName(manip);
      ds->SetTitle(fEnv.GetValueOf(manip, "DataSet.Title").Default("Experimental dataset"));
      ds->SetDataPathSubdir(fEnv.GetValueOf(manip, "DataSet.RepositoryDir").Default(manip));
      ds->SetRepository(fRepository);
      fDataSets.Add(ds);
 
   }
 
   return kTRUE;
}
 
 
 
 
Bool_t KVDataSetManager::ReadTaskList()
{
   //Initialise list of all known analysis tasks from informations in .kvrootrc config files
   //(and user's .kvrootrc)
 
   KVString task_list = fEnv.GetValueOf("DataAnalysisTask").Default("");
 
   fTasks.Clear();
 
   task_list.Begin(" ");
   while (!task_list.End()) {
 
      KVDataAnalysisTask* dat = new KVDataAnalysisTask;
      auto name = task_list.Next();
      dat->SetName(name);
      dat->SetTitle(fEnv.GetValueOf(name, "DataAnalysisTask.Title").Default(""));
      dat->SetPrereq(fEnv.GetValueOf(name, "DataAnalysisTask.Prereq").Default(""));
      dat->SetOutputDataType(fEnv.GetValueOf(name, "DataAnalysisTask.Output").Default(""));
      dat->SetDataAnalyser(fEnv.GetValueOf(name, "DataAnalysisTask.Analyser").Default("KVDataAnalyser"));
      dat->SetWithUserClass(fEnv.GetValueOf(name, "DataAnalysisTask.UserClass").Default(kFALSE));
      dat->SetUserBaseClass(fEnv.GetValueOf(name, "DataAnalysisTask.UserClass.Base").Default(""));
      dat->SetStatusUpdateInterval(fEnv.GetValueOf(name, "DataAnalysisTask.StatusUpdateInterval").Default(1000));
      fTasks.Add(dat);
 
   }
 
   return kTRUE;
}
 
 
 
 
void KVDataSetManager::Print(Option_t* opt) const
{
   //Print list of datasets:
   //
   //  + if `opt=""` (default) all datasets are shown with full information
   //  + if `opt="available"` only available datasets are shown, each with a number which can
   //     be used with GetAvailableDataSet() in order to retrieve the corresponding dataset.
 
   TString Sopt(opt);
   Sopt.ToUpper();
   if (Sopt.BeginsWith("AVAIL")) {
      if (!fNavailable) {
         cout << "                    *** No available datasets ***" <<
              endl;
         return;
      }
      else {
         for (int i = 1; i <= fNavailable; i++) {
            KVDataSet* ds = GetAvailableDataSet(i);
            cout << "\t" << i << ". " << ds->GetTitle() << endl;
         }
      }
      return;
   }
   if (fDataSets.GetSize()) {
      TIter next(&fDataSets);
      KVDataSet* ds;
      while ((ds = (KVDataSet*) next()))
         ds->ls();
   }
}
 
 
 
 
void KVDataSetManager::CheckAvailability()
{
   //Check availability of datasets in repository associated to this data set manager
   //
   //If caching is activated for the parent repository, i.e. if
   //~~~
   //    [repository name].DataRepository.CacheAvailable:    yes
   //~~~
   //then instead of directly checking the existence of the directories for each dataset,
   //we use the cached information written in the file
   //`$HOME/.kaliveda/[repository name].available.datasets`
   //unless
   //  + (1) it doesn't exist, or
   //  + (2) the file is older than the maximum cache time (in seconds) defined by
   //~~~
   //   [repository name].DataRepository.MaxCacheSeconds:
   //~~~
   //
   //In either of these 2 cases, we check the existence of the directories and update/
   //create the cache file.
   //
   //If the repository appears to be empty (perhaps because we are using a remote access
   //protocol to check it, and the protocol has some problems...), then as a last resort we
   //we will use the cache if it exists, whatever its age.
 
   if (fCacheAvailable) {
      //caching of dataset availability is activated
      if (CheckCacheStatus()) {
         //cache file exists and is not out of date
         if (ReadAvailableDatasetsFile()) return;
      }
   }
 
   // print (repository-dependent) warning/informational message
   if (fRepository) fRepository->PrintAvailableDatasetsUpdateWarning();
 
   //open temporary file
   ofstream tmp_file;
   TString tmp_file_path = fCacheFileName;
   KVBase::OpenTempFile(tmp_file_path, tmp_file);
 
   fNavailable = 0;
   if (fDataSets.GetSize()) {
      TIter next(&fDataSets);
      KVDataSet* ds;
      while ((ds = (KVDataSet*) next())) {
         //The results of this check are written in $KVROOT/KVFiles/[repository name].available.datasets
         //This file may be read by KVRemoteDataSetManager::CheckAvailability when this
         //data repository is accessed as a remote data repository from a remote machine.
         //In this case we do not want the identity of the user to influence the contents of the file.
         //Therefore even for 'unavailable' datasets we write the available datatypes (if any)
         //in the file.
         tmp_file << ds->GetName() << " : ";
         ds->CheckAvailable();
         tmp_file << ds->GetAvailableDataTypes() << endl;
         if (ds->IsAvailable()) {
            fNavailable++;
         }
      }
 
      //close temp file
      tmp_file.close();
      //if datasets are found, then we copy the temporary file to KVFiles directory,
      //overwriting any previous version. if no datasets were found, we try the cache
      //file (if it exists)
      if (fNavailable && fRepository) { //if no repository is associated, no need to keep file
         TString runlist = KVBase::GetWORKDIRFilePath(fCacheFileName.Data());
         gSystem->CopyFile(tmp_file_path, runlist, kTRUE);
         //set access permissions to 664
         gSystem->Chmod(runlist.Data(), 0664);
      }
 
      //delete temp file
      gSystem->Unlink(tmp_file_path);
 
      if (!fNavailable) {
         //no datasets found when checking file system ?
         //can we rely on the cache file ?
         ReadAvailableDatasetsFile();
      }
      else {
         //now set up array of available datasets' indices
         fIndex.clear();
         next.Reset();
         Int_t j(0);
         while ((ds = (KVDataSet*) next())) {
            if (ds->IsAvailable()) {
               fIndex.push_back(j);
            }
            j++;
         }
      }
   }
}
 
 
 
 
 
KVDataSet* KVDataSetManager::GetDataSet(Int_t index) const
{
   //Return pointer to DataSet using index in list of all datasets, index>=0
   if (fDataSets.GetSize() && index < fDataSets.GetSize())
      return (KVDataSet*) fDataSets.At(index);
   return 0;
}
 
 
 
 
KVDataSet* KVDataSetManager::GetDataSet(const Char_t* name)
{
   //Return pointer to DataSet using name
   return (KVDataSet*) fDataSets.FindObjectByName(name);
}
 
 
 
 
KVDataSet* KVDataSetManager::GetAvailableDataSet(Int_t index) const
{
   //Return pointer to available DataSet using index of available datasets
   //
   //Note this index begins at 1, and corresponds to the number printed next to the dataset
   //when Print("available") is called
   if (fNavailable && index && index <= fNavailable)
      return GetDataSet(fIndex[index - 1]);
   return 0;
}
 
 
 
 
KVDataAnalysisTask* KVDataSetManager::GetTask(const Char_t* name)
{
   //Return pointer to named data analysis task
   return (KVDataAnalysisTask*) fTasks.FindObjectByName(name);
}
 
 
 
 
KVDataSet* KVDataSetManager::NewDataSet()
{
   //Creates and returns pointer to new data set object
   return (new KVDataSet);
}
 
 
 
 
void KVDataSetManager::Update()
{
   //Called when the physical state of the repository has changed i.e. a subdirectory for
   //a new dataset or datatype has been added or removed. We update the available datatsets,
   //datatypes and analysis tasks.
 
   //check which datasets are available
   CheckAvailability();
 
   //set available data analysis tasks for available datasets
   if (GetNavailable()) {
      for (Int_t i = 1; i <= GetNavailable(); i++)
         GetAvailableDataSet(i)->SetAnalysisTasks(&fTasks);
   }
}
 
 
 
 
Bool_t KVDataSetManager::OpenAvailableDatasetsFile()
{
   //Opens file KVBase::WorkingDirectory()/[repository name].available.datasets
   //containing cached info on available datasets and
   //associated subdirectories in data repository.
   //Opens file for reading, & if all goes well returns kTRUE.
   //Returns kFALSE in case of problems.
 
   return KVBase::SearchAndOpenKVFile(KVBase::GetWORKDIRFilePath(fCacheFileName), fDatasets);
}
 
 
 
 
Bool_t KVDataSetManager::ReadAvailableDatasetsFile()
{
   //Opens and reads file containing cached info on available datasets, and sets
   //the availability of the concerned datasets.
   //Returns kTRUE if all goes well.
   //Returns kFALSE if no cache exists or if file cannot be opened.
   if (OpenAvailableDatasetsFile()) {
      Info("ReadAvailableDataSetsFile",
           "Reading cached information in file %s", fCacheFileName.Data());
      //read file
      KVString line;
      line.ReadLine(fDatasets);
      while (fDatasets.good()) {
 
         line.Begin(": ,");
 
         //first entry is dataset name
         TString datasetname = line.Next(kTRUE);
         KVDataSet* dataset = GetDataSet(datasetname.Data());
 
         if (dataset) { //check dataset is known to local version of KaliVeda
            //in case of remote repository, there may be datasets in the remote repository which are not defined here
            if (!line.End()) {
               //AVAILABLE DATASET
               dataset->SetAvailable();
               fNavailable++;
               while (!line.End()) {
                  //each following entry is a subdirectory name
                  dataset->AddAvailableDataType(line.Next(kTRUE));
               }
            }
            else {
               //UNAVAILABLE DATASET (no subdirs)
               dataset->SetAvailable(kFALSE);
            }
         }
         line.ReadLine(fDatasets);
      }
 
      //close file
      fDatasets.close();
      fDatasets.clear();
 
      if (fNavailable) {
         TIter next(&fDataSets);
         //now set up array of available datasets' indices
         fIndex.clear();
         Int_t j(0);
         KVDataSet* ds;
         while ((ds = (KVDataSet*) next())) {
            if (ds->IsAvailable()) {
               fIndex.push_back(j);
            }
            j++;
         }
      }
      //all is OK
      return kTRUE;
   }
   //we could not find/open the cache file
   return kFALSE;
}
 
 
 
 
 
Bool_t KVDataSetManager::CheckCacheStatus()
{
   //We check the status of the available datasets cache file.
   //\return kTRUE if the file exists & was last modified less than fMaxCacheTime seconds ago.
 
   TString fullpath;
   Info("KVDataSetManager::CheckCacheStatus", "Checking for available datasets cache file...");
   if (KVBase::SearchKVFile(KVBase::GetWORKDIRFilePath(fCacheFileName), fullpath)) {
 
      // file exists - how old is it ?
      FileStat_t file_info;
      gSystem->GetPathInfo(fullpath.Data(), file_info);
      TDatime file_date(file_info.fMtime);
      TDatime now;
      UInt_t file_age = now.Convert() - file_date.Convert();
      Info("KVDataSetManager::CheckCacheStatus", "...file found. It is %u seconds old", file_age);
      if (file_age < fMaxCacheTime) {
         Info("KVDataSetManager::CheckCacheStatus", "Using cached file");
         return kTRUE;
      }
      else
         Info("KVDataSetManager::CheckCacheStatus", "File is too old (max time=%u). Update will be performed.", fMaxCacheTime);
   }
   else
      Info("KVDataSetManager::CheckCacheStatus", "...no file found");
   return kFALSE;
}
 
 
 
 
KVDataAnalysisTask* KVDataSetManager::GetAnalysisTaskAny(const Char_t* keywords) const
{
   // This method returns a pointer to the analysis task whose description (title) contains
   // all of the whitespace-separated keywords (which may be regular expressions)
   // given in the string "keywords". The comparison is case-insensitive.
 
   //case-insensitive search for matches in list of all analysis tasks, based on 'title' attribute
   return (KVDataAnalysisTask*)GetAnalysisTaskList()->FindObjectAny("title", keywords, kTRUE, kFALSE);
}