/home/mhx/git/github/libmoost/include/moost/container/neigh_multi_map.hpp

Go to the documentation of this file.

/* vim:set ts=3 sw=3 sts=3 et: */
#ifndef __NEIGH_MULTI_MAP_CONTAINER_H
#define __NEIGH_MULTI_MAP_CONTAINER_H

#include <string>
#include <vector>
#include <algorithm>
#include <limits>
#include <fstream>
#include <stdexcept>

#include <boost/cstdint.hpp>

#include "multi_map.hpp"
#include "dense_hash_map.hpp"

namespace moost { namespace container {

// -----------------------------------------------------------------------------

typedef std::pair<int, float> entry_type;
struct IdentityPolicy
{
   inline bool operator()(int entry, int& res) const
   {
      res = entry;
      return true;
   }
};

// -----------------------------------------------------------------------------

template < typename TLocMap = moost::container::dense_hash_map<int, multimap_value_type> >
class neigh_multi_map :
   public multi_map< int, std::pair<int, float>, TLocMap >
{
public:

   typedef typename multi_map< int, std::pair<int, float>, TLocMap >::loc_map_policy_type loc_map_policy_type;

public:

   //neigh_multi_map( const loc_map_policy_type& locHandlerPolicy = loc_map_policy_type() )
   //   : multi_map<int, std::pair<int, float>, TLocMap>(locHandlerPolicy)
   //{}

   // the default policy uses dense hash map, and the default empty key is 0,
   // so if your data has zeroes for keys you have to make sure
   // you are using a different policy, i.e.
   // neigh_multi_map<> m( neigh_multi_map<>::loc_map_policy_type(-1) );
   neigh_multi_map(const loc_map_policy_type& locHandlerPolicy )
      : multi_map<int, std::pair<int, float>, TLocMap>(locHandlerPolicy), m_externalLocMap(true)
   {}

   neigh_multi_map()
      : m_externalLocMap(false)
   {}


   // WhichKey is either 1 or 2
   // Expect to be sorted by key
   void create_map( const std::string& dataFileName,
                    int maxEntriesPerVec = (std::numeric_limits<int>::max)() )
   {
      IdentityPolicy ip;
      create_map(dataFileName, ip, maxEntriesPerVec);
   }

   // WhichKey is either 1 or 2
   // Expect to be sorted by key

   template <typename TranformIDPolicy>
   void create_map( const std::string& dataFileName,
                    const TranformIDPolicy& getIDPolicy,
                    int maxEntriesPerVec = (std::numeric_limits<int>::max)() );

   inline void create_map_from_vector( std::vector<std::pair<int, std::pair<int, float> > >& i2i )
   {
      multi_map< int, std::pair<int, float>, TLocMap >::template create_map<1>(i2i.begin(), i2i.end());
   }

private:

   using multi_map< int, std::pair<int, float>, TLocMap >::m_data;
   using multi_map< int, std::pair<int, float>, TLocMap >::m_locations;

   using multi_map< int, std::pair<int, float>, TLocMap >::m_locHandlerPolicy;

   bool m_externalLocMap;

   // -----------------------------------------------------------------------------

#ifdef _WIN32
   // In order to seek with large files in windows use
   // this function to "extract" the file pointer from
   // a stream:
   //
   // FILE* fp = getFilePointer(fileStream.rdbuf());
   //
   // then use the 64bits seek function:
   //
   // _fseeki64( fp, pos, SEEK_SET);
   //
   inline FILE* getFilePointer(std::filebuf* pFileBuf)
   {
      return reinterpret_cast<FILE*>(
         *reinterpret_cast<FILE**>(
         reinterpret_cast<char*>(pFileBuf)+76 ) );
   }
#endif

};

// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------

template <typename TLocMap>
template <typename TranformIDPolicy>
void neigh_multi_map<TLocMap>::create_map( const std::string& dataFileName,
                                           const TranformIDPolicy& getIDPolicy,
                                           int maxEntriesPerVec /*= (std::numeric_limits<int>::max)() */ )
{
   std::ifstream fileSource(dataFileName.c_str(), std::ios::binary);
   if ( !fileSource.is_open() )
      throw std::runtime_error("Cannot open file <" + dataFileName + ">!");

#ifdef _WIN32
   FILE* fp = getFilePointer(fileSource.rdbuf());
#endif

   int numKeys;
   fileSource.read( reinterpret_cast<char*>( &numKeys ), sizeof(int) );
   if ( fileSource.eof() )
      throw std::runtime_error("Empty source on <" + dataFileName + ">!");

   // scanning
   boost::int64_t currPos = sizeof(int);
   int tmpID, numEntries;
   int numToRead;
   int totEntries = 0;

   for (int i = 0;; ++i)
   {
      fileSource.read( reinterpret_cast<char*>(&tmpID), sizeof(int) );
      if ( fileSource.eof() )
         break;
      fileSource.read( reinterpret_cast<char*>( &numEntries ), sizeof(int) );
      numToRead = (std::min)( numEntries, maxEntriesPerVec );
      totEntries += numToRead;

      currPos += sizeof(int) + // tmpID
                 sizeof(int) + // numEntries
                 numEntries * sizeof(std::pair<int, float>);

#ifdef _WIN32
      _fseeki64( fp, currPos, SEEK_SET);
#else
      fileSource.seekg( currPos, std::ios::beg );
#endif
   }

   // we've got the number!
   if ( totEntries == 0 )
      throw std::runtime_error("Empty source on <" + dataFileName + ">!");

   // allocating
   this->m_data.resize(totEntries);

   if ( !m_externalLocMap )
      m_locHandlerPolicy.resize(this->m_locations, numKeys);
   //TLocHandler::reserve(this->m_locations, numKeys);
   //this->m_locations.resize(numKeys);


   // now loading
   fileSource.clear();
   currPos = sizeof(int);

#ifdef _WIN32
   fileSource.seekg( static_cast<long>(currPos), std::ios::beg );
#else
   fileSource.seekg( currPos, std::ios::beg );
#endif
   int entryPos = 0;

   int transformedID;

   for (int i = 0;; ++i)
   {
      fileSource.read( reinterpret_cast<char*>(&tmpID), sizeof(int) );
      if ( fileSource.eof() )
         break;

      fileSource.read( reinterpret_cast<char*>( &numEntries ), sizeof(int) );
      numToRead = (std::min)( numEntries, maxEntriesPerVec );

      if ( numToRead == 0 )
         continue;

      if ( entryPos + numToRead > totEntries )
         throw std::runtime_error("There were more entries than what was found during scan!");

      fileSource.read( reinterpret_cast<char*>( &(m_data[entryPos]) ), numToRead * sizeof(entry_type) );

      if ( getIDPolicy(tmpID, transformedID) )
         m_locations[transformedID] = std::make_pair(entryPos, numToRead);

      entryPos += numToRead;
      currPos += sizeof(int) + // tmpID
                 sizeof(int) + // numEntries
                 numEntries * sizeof(std::pair<int, float>);

#ifdef _WIN32
      _fseeki64( fp, currPos, SEEK_SET);
#else
      fileSource.seekg( currPos, std::ios::beg );
#endif
   }
}

// -----------------------------------------------------------------------------

}}

#endif // __NEIGH_MULTI_MAP_CONTAINER_H

// -----------------------------------------------------------------------------