classencoder.h

Go to the documentation of this file.

#ifndef CLASSENCODER_H
#define CLASSENCODER_H

/*****************************
* Colibri Core
*   by Maarten van Gompel
*   Centre for Language Studies
*   Radboud University Nijmegen
*
*   http://proycon.github.io/colibri-core
*   
*   Licensed under GPLv3
*****************************/
#if HAVE_CONFIG_H
#include <config.h>
#endif
#include <unordered_map>
#include <string>
#include <vector>
#include <fstream>
#include <pattern.h>
#include <common.h>

#ifdef WITHFOLIA
#include "libfolia/document.h"
#include "libfolia/folia.h"
#endif

class ClassEncoder {
    private:
     std::unordered_map<std::string,unsigned int> classes;
     unsigned int highestclass;
     unsigned int minlength;
     unsigned int maxlength;
    public:
     static const unsigned char delimiterclass = 0;
     static const unsigned char unknownclass = 2;
     static const unsigned char skipclass = 3;
     static const unsigned char flexclass = 4;
    ClassEncoder(const unsigned int minlength = 0, const unsigned int maxlength = 0);

    ClassEncoder(const std::string & filename, const unsigned int minlength = 0, const unsigned int maxlength = 0); //load an existing classer

    void load(const std::string & filename, const unsigned int minlength = 0, const unsigned int maxlength = 0); //load an existing classer
    
    void build(const std::string & filename, unsigned int threshold=0); 

    void build(std::vector<std::string> & files, bool quiet=false, unsigned int threshold =0); 
    
    //auxiliary functions called by build: first do processcorpus() for each
    //corpus, then call buildclasses() once when done:
    //
    
    void buildclasses(const std::unordered_map<std::string,unsigned int> & freqlist, unsigned int threshold =0);

    void processcorpus(const std::string & filename, std::unordered_map<std::string,unsigned int> & freqlist);
    void processcorpus(std::istream * in, std::unordered_map<std::string,unsigned int> & freqlist);
    #ifdef WITHFOLIA

    void processfoliacorpus(const std::string & filename, std::unordered_map<std::string,unsigned int> & freqlist);
    #endif

    std::unordered_map<unsigned int, std::string> added;
    

    int outputlength(const std::string & line);

    int encodestring(const std::string & line, unsigned char * outputbuffer, bool allowunknown, bool autoaddunknown=false);

    void encodefile(const std::string & inputfilename, const std::string & outputfilename, bool allowunknown, bool autoaddunknown=false,  bool append=false, bool quiet=false);
    void encodefile(std::istream * IN, std::ostream * OUT, bool allowunknown, bool autoaddunknown, bool quiet=false, bool append=false);

    std::vector<unsigned int> encodeseq(const std::vector<std::string> & seq);
    
    Pattern buildpattern(const std::string & patternstring, bool allowunknown=false, bool autoaddunknown = false);  //not thread-safe
    Pattern buildpattern_safe(const std::string &  patternstring, bool allowunknown=false, bool autoaddunknown = false);  //thread-safe

 
    void add(const std::string &, const unsigned int cls);
    
    unsigned int gethighestclass() { return highestclass; } 
    
    void save(const std::string & filename);
    
    int size() const {
        return classes.size();
    }
    
    unsigned int operator[](const std::string & key) {
         return classes[key];
    }

    typedef std::unordered_map<std::string, unsigned int>::const_iterator const_iterator;

    const_iterator begin() const {
        return classes.begin();
    }
    const_iterator end() const {
        return classes.end();
    }
};    

unsigned int  inttobytes(unsigned char * buffer, unsigned int cls);
unsigned char * inttobytes_v1(unsigned int, int & length);
int readline(std::istream* IN, unsigned char* buffer, const int);

unsigned char * convert_v1_v2(const unsigned char * olddata, unsigned int & newlength);
unsigned char * convert_v1_v2(std::istream * in, bool ignoreeol, bool debug);

const int countwords(const unsigned char* data, const int l);
#endif