translation/tools/mosesdecoder/moses/TranslationModel/CompactPT/MonotonicVector.h

// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#ifndef moses_MonotonicVector_h
#define moses_MonotonicVector_h

// MonotonicVector - Represents a monotonic increasing function that maps
// positive integers of any size onto a given number type. Each value has to be
// equal or larger than the previous one. Depending on the stepSize it can save
// up to 90% of memory compared to a std::vector<long>. Time complexity is roughly
// constant, in the worst case, however, stepSize times slower than a normal
// std::vector.

#include <vector>
#include <limits>
#include <algorithm>
#include <cstdio>
#include <cassert>

#include "ThrowingFwrite.h"
#include "ListCoders.h"
#include "MmapAllocator.h"

namespace Moses
{

template<typename PosT = size_t, typename NumT = size_t, PosT stepSize = 32,
         template <typename> class Allocator = std::allocator>
class MonotonicVector
{
private:
  typedef std::vector<NumT, Allocator<NumT> > Anchors;
  typedef std::vector<unsigned int, Allocator<unsigned int> > Diffs;

  Anchors m_anchors;
  Diffs m_diffs;
  std::vector<unsigned int> m_tempDiffs;

  size_t m_size;
  PosT m_last;
  bool m_final;

public:
  typedef PosT value_type;

  MonotonicVector() : m_size(0), m_last(0), m_final(false) {}

  size_t size() const {
    return m_size + m_tempDiffs.size();
  }

  PosT at(size_t i) const {
    PosT s = stepSize;
    PosT j = m_anchors[i / s];
    PosT r = i % s;

    typename Diffs::const_iterator it = m_diffs.begin() + j;

    PosT k = 0;
    k += VarInt32::DecodeAndSum(it, m_diffs.end(), 1);
    if(i < m_size)
      k += Simple9::DecodeAndSum(it, m_diffs.end(), r);
    else if(i < m_size + m_tempDiffs.size())
      for(size_t l = 0; l < r; l++)
        k += m_tempDiffs[l];

    return k;
  }

  PosT operator[](PosT i) const {
    return at(i);
  }

  PosT back() const {
    return at(size()-1);
  }

  void push_back(PosT i) {
    assert(m_final != true);

    if(m_anchors.size() == 0 && m_tempDiffs.size() == 0) {
      m_anchors.push_back(0);
      VarInt32::Encode(&i, &i+1, std::back_inserter(m_diffs));
      m_last = i;
      m_size++;

      return;
    }

    if(m_tempDiffs.size() == stepSize-1) {
      Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(),
                      std::back_inserter(m_diffs));
      m_anchors.push_back(m_diffs.size());
      VarInt32::Encode(&i, &i+1, std::back_inserter(m_diffs));

      m_size += m_tempDiffs.size() + 1;
      m_tempDiffs.clear();
    } else {
      PosT last = m_last;
      PosT diff = i - last;
      m_tempDiffs.push_back(diff);
    }
    m_last = i;
  }

  void commit() {
    assert(m_final != true);
    Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(),
                    std::back_inserter(m_diffs));
    m_size += m_tempDiffs.size();
    m_tempDiffs.clear();
    m_final = true;
  }

  size_t usage() {
    return m_diffs.size() * sizeof(unsigned int)
           + m_anchors.size() * sizeof(NumT);
  }

  size_t load(std::FILE* in, bool map = false) {
    size_t byteSize = 0;

    byteSize += fread(&m_final, sizeof(bool), 1, in) * sizeof(bool);
    byteSize += fread(&m_size, sizeof(size_t), 1, in) * sizeof(size_t);
    byteSize += fread(&m_last, sizeof(PosT), 1, in) * sizeof(PosT);

    byteSize += loadVector(m_diffs, in, map);
    byteSize += loadVector(m_anchors, in, map);

    return byteSize;
  }

  template <typename ValueT>
  size_t loadVector(std::vector<ValueT, std::allocator<ValueT> >& v,
                    std::FILE* in, bool map = false) {
    // Can only be read into memory. Mapping not possible with std:allocator.
    assert(map == false);

    size_t byteSize = 0;

    size_t valSize;
    byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t);

    v.resize(valSize, 0);
    byteSize += std::fread(&v[0], sizeof(ValueT), valSize, in) * sizeof(ValueT);

    return byteSize;
  }

  template <typename ValueT>
  size_t loadVector(std::vector<ValueT, MmapAllocator<ValueT> >& v,
                    std::FILE* in, bool map = false) {
    size_t byteSize = 0;

    size_t valSize;
    byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t);

    if(map == false) {
      // Read data into temporary file (default constructor of MmapAllocator)
      // and map memory onto temporary file. Can be resized.

      v.resize(valSize, 0);
      byteSize += std::fread(&v[0], sizeof(ValueT), valSize, in) * sizeof(ValueT);
    } else {
      // Map it directly on specified region of file "in" starting at valPos
      // with length valSize * sizeof(ValueT). Mapped region cannot be resized.

      size_t valPos = std::ftell(in);

      Allocator<ValueT> alloc(in, valPos);
      std::vector<ValueT, Allocator<ValueT> > vTemp(alloc);
      vTemp.resize(valSize);
      v.swap(vTemp);

      std::fseek(in, valSize * sizeof(ValueT), SEEK_CUR);
      byteSize += valSize * sizeof(ValueT);
    }

    return byteSize;
  }

  size_t save(std::FILE* out) {
    if(!m_final)
      commit();

    bool byteSize = 0;
    byteSize += ThrowingFwrite(&m_final, sizeof(bool), 1, out) * sizeof(bool);
    byteSize += ThrowingFwrite(&m_size, sizeof(size_t), 1, out) * sizeof(size_t);
    byteSize += ThrowingFwrite(&m_last, sizeof(PosT), 1, out) * sizeof(PosT);

    size_t size = m_diffs.size();
    byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
    byteSize += ThrowingFwrite(&m_diffs[0], sizeof(unsigned int), size, out) * sizeof(unsigned int);

    size = m_anchors.size();
    byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t);
    byteSize += ThrowingFwrite(&m_anchors[0], sizeof(NumT), size, out) * sizeof(NumT);

    return byteSize;
  }

  void swap(MonotonicVector<PosT, NumT, stepSize, Allocator> &mv) {
    if(!m_final)
      commit();

    m_diffs.swap(mv.m_diffs);
    m_anchors.swap(mv.m_anchors);
  }
};

}
#endif