/home/mhx/git/github/libmoost/include/moost/io/file_backed_data_source.hpp

Go to the documentation of this file.

/* vim:set ts=3 sw=3 sts=3 et: */
#ifndef MOOST_IO_FILE_BACKED_DATA_SOURCE_HPP__
#define MOOST_IO_FILE_BACKED_DATA_SOURCE_HPP__

#include <string>
#include <vector>
#include <map>
#include <stdexcept>

#include <boost/bind.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/algorithm/string.hpp>

#include "../safe_shared_ptr.hpp"
#include "file_watcher.hpp"
#include "../terminal_format.hpp"
#include "../xml/simple_parser.hpp"
#include "../logging/class_logger.hpp"

namespace moost { namespace io {

template <typename T>
class data_policy_base
{
public:
   typedef T data_type;
   virtual ~data_policy_base() { }
   virtual std::string getName() const = 0;
   virtual boost::shared_ptr<T> loadFromFile(const std::string& filepath) const = 0;
   virtual size_t size(boost::shared_ptr<T> pData) const = 0;
};


class loadable
{
public:
   virtual ~loadable() { }
   virtual void load() = 0;
};


struct file_backed_data_source_config
{
public:
   std::string filepath;            // the filepath of the data file
   int minSecsSinceLastLoad;        // reject reload sooner than this
   bool throwOnFirstLoadFail;       // if true, throw an exception if the initial load fails
   double minProportionOfLastLoad;  //  reject reload smaller than this

   // some possible default values, you still need to set the filepath!
   file_backed_data_source_config() :
   minSecsSinceLastLoad(30), throwOnFirstLoadFail(true), minProportionOfLastLoad(0.5)
   {
   }

   file_backed_data_source_config(const std::string& path, int minSecs, bool throwOnFirst,
      double minProportion) :
   filepath(path), minSecsSinceLastLoad(minSecs), throwOnFirstLoadFail(throwOnFirst),
   minProportionOfLastLoad(minProportion)
   {
   }
};

class file_backed_data_source_config_factory
{
public:
   file_backed_data_source_config createFromXml(const std::string& xmlFilepath)
   {
      tag_map_t tagMap;
      xmlToMap(tagMap, xmlFilepath);
      file_backed_data_source_config conf;
      setAndRemove(conf.filepath, "Filepath", tagMap);
      setAndRemove(conf.minSecsSinceLastLoad, "MinSecsSinceLastLoad", tagMap);
      setAndRemove(conf.throwOnFirstLoadFail, "ThrowOnFirstLoadFail", tagMap);
      setAndRemove(conf.minProportionOfLastLoad, "MinProportionOfLastLoad", tagMap);
      if (!tagMap.empty())
         throw std::runtime_error("Unexpected xml tag: " + tagMap.begin()->first);
      return conf;
   }

private:
   typedef std::map<std::string, std::string> tag_map_t;

   void xmlToMap(std::map<std::string, std::string>& tagMap, const std::string& xmlFilepath)
   {
      moost::xml::simple_parser xmlParser;
      xmlParser.load(xmlFilepath);
      const moost::xml::simple_parser::tree_branch_t& root = xmlParser.get_root();
      if (!root.front())
         throw std::runtime_error("No xml root tag");
      std::string rootTag(root.front()->header);
      if (rootTag != "FileBackedDataSource")
         throw std::runtime_error("Unexpected xml root tag: " + rootTag);
      root.front()->leaves2map(tagMap);
   }

   template <typename T>
   void setAndRemove(T& value, const std::string& name, tag_map_t& tagMap)
   {
      tag_map_t::iterator it = tagMap.find(name);
      if (it == tagMap.end())
         return;
      value = boost::lexical_cast<T>(it->second);
      tagMap.erase(it);
   }

   void setAndRemove(bool& value, const std::string& name, tag_map_t& tagMap)
   {
      tag_map_t::iterator it = tagMap.find(name);
      if (it == tagMap.end())
         return;
      std::string lowerVal = boost::to_lower_copy(it->second);
      if (lowerVal == "true")
         value = true;
      else if (lowerVal == "false")
         value = false;
      else
         throw std::runtime_error("Unexpected value for boolean tag: " + lowerVal);
      tagMap.erase(it);
   }
};

template <typename DataPolicy>
class file_backed_data_source : public loadable
{
public:

   typedef typename DataPolicy::data_type data_type;

   file_backed_data_source(const DataPolicy& dataPolicy) : m_dataPolicy(dataPolicy),
      m_firstLoad(true), m_lastLoadTime(-1) { }

   void configure(file_backed_data_source_config conf)
   {
      m_conf = conf;
   }

   const file_backed_data_source_config& getConfig() const
   {
      return m_conf;
   }

   size_t size() const
   {
      boost::shared_ptr<data_type> pData = m_pData.get_shared();
      return m_dataPolicy.size(pData);
   }

   int getLastLoadTime() const
   {
      return m_lastLoadTime;
   }

   void registerLoadable(boost::shared_ptr<loadable> pOther, bool loadOtherFirst = true)
   {
      if (loadOtherFirst)
         m_preRegistered.push_back(pOther);
      else
         m_postRegistered.push_back(pOther);
   }

   boost::shared_ptr<data_type> get_shared_ptr()
   {
      return m_pData.get_shared();
   }

   void load()
   {
      if (m_firstLoad)
      {
         m_fileWatcher.start();
         m_fileWatcher.insert(m_conf.filepath, boost::bind(&file_backed_data_source<DataPolicy>::reload, this, _1, _2));
      }
      reload(file_watcher::CHANGED, m_conf.filepath);
      if (m_firstLoad)
      {
         m_firstLoad = false;
      }
   }

private:
   DataPolicy m_dataPolicy;

   file_backed_data_source_config m_conf;
   bool m_firstLoad;
   int m_lastLoadTime;

   file_watcher m_fileWatcher;

   moost::safe_shared_ptr<data_type> m_pData;

   typedef std::vector<boost::shared_ptr<loadable> > registered_t;
   registered_t m_preRegistered;
   registered_t m_postRegistered;

   void reload(file_watcher::file_action action, const std::string& filepath)
   {
      if (action != file_watcher::CHANGED)
         return;

      // abandon reload if it's too soon after previous one
      int timeNow = static_cast<int>(time(NULL));
      if (!m_firstLoad && timeNow < m_lastLoadTime + m_conf.minSecsSinceLastLoad)
         return;

      // force pre-registered sources to reload first
      for (registered_t::iterator it = m_preRegistered.begin(); it != m_preRegistered.end(); ++it)
         (*it)->load();

      MLOG_CLASS_INFO("Updating " << m_dataPolicy.getName() << "..");

      boost::shared_ptr<data_type> pData;
      loadWithErrorHandling(pData, filepath);

      // abandon new dataset if it looks too small
      size_t newSize = m_dataPolicy.size(pData);
      if (!m_firstLoad && newSize < size() * m_conf.minProportionOfLastLoad)
         return;

      MLOG_CLASS_INFO(moost::terminal_format::getOkay() << ": Loaded " << newSize);

      m_pData = pData;
      m_lastLoadTime = static_cast<int>(time(NULL));

      // force post-registered sources to reload afterwards
      for (registered_t::iterator it = m_postRegistered.begin(); it != m_postRegistered.end(); ++it)
         (*it)->load();

   }

   void loadWithErrorHandling(boost::shared_ptr<data_type>& pData, const std::string& filepath)
   {
      try
      {
         pData = m_dataPolicy.loadFromFile(filepath);
      }
      catch (std::runtime_error& ex)
      {
         MLOG_CLASS_WARN(ex.what() << " loading " << m_dataPolicy.getName()
            << " from " << filepath);
         if (m_firstLoad && m_conf.throwOnFirstLoadFail)
         {
            throw;
         }
      }
      catch (...)
      {
         MLOG_CLASS_WARN("exception loading " << m_dataPolicy.getName()
            << " from " << filepath);
         if (m_firstLoad && m_conf.throwOnFirstLoadFail)
         {
            throw;
         }
      }
   }
};

class file_backed_data_source_factory
{
public:

   template <typename DataPolicy>
   boost::shared_ptr<file_backed_data_source<DataPolicy> > createFromConfig(const DataPolicy& dataPolicy, file_backed_data_source_config conf)
   {
      boost::shared_ptr<file_backed_data_source<DataPolicy> > pSource(new file_backed_data_source<DataPolicy>(dataPolicy));
      pSource->configure(conf);

      return pSource;
   }

   template <typename DataPolicy>
   boost::shared_ptr<file_backed_data_source<DataPolicy> > createFromXml(const DataPolicy& dataPolicy, const std::string& xmlFilepath)
   {
      file_backed_data_source_config_factory config_factory;
      file_backed_data_source_config conf = config_factory.createFromXml(xmlFilepath);
      return createFromConfig(dataPolicy, conf);
   }
};

}}

#endif