pattern.h

Go to the documentation of this file.

#ifndef COLIBRIPATTERN_H
#define COLIBRIPATTERN_H

/*****************************
* Colibri Core
*   by Maarten van Gompel
*   Centre for Language Studies
*   Radboud University Nijmegen
*
*   http://proycon.github.io/colibri-core
*   
*   Licensed under GPLv3
*****************************/

#include <string>
#include <iostream>
#include <ostream>
#include <istream>
#include <unordered_map>
#include <vector>
#include <set>
#include <map>
#include <array>
#include <unordered_set>
#include <iomanip> // contains setprecision()
#include <exception>
#include <algorithm>
#include <climits>
#include "common.h"
#include "classdecoder.h"


const int MAINPATTERNBUFFERSIZE = 40960;

enum PatternCategory {
    UNKNOWNPATTERN = 0, 
    NGRAM = 1, 
    SKIPGRAM = 2, 
    FLEXGRAM = 3, 
};

enum PatternType {
    PATTERN = 0,
    PATTERNPOINTER = 1,
};

void readanddiscardpattern(std::istream * in, bool pointerformat = false);
int reader_passmarker(const unsigned char c, std::istream * in); 



class PatternPointer;

class Pattern {
    public:
     const static int patterntype = PATTERN;
     unsigned char * data; 
     Pattern() { data = new unsigned char[1]; data[0] = ClassDecoder::delimiterclass; }

     Pattern(const unsigned char* dataref, const int size); 

     Pattern(const Pattern& ref, unsigned int begin, unsigned int length); 
     Pattern(const PatternPointer& ref,unsigned int begin, unsigned int length);

     Pattern(const Pattern& ref);
     Pattern(const PatternPointer& ref);

     Pattern(std::istream * in, bool ignoreeol = false, const unsigned char version = 2, const unsigned char * corpusstart = NULL, bool debug = false); 
     //Pattern(std::istream * in, unsigned char * buffer, int maxbuffersize, bool ignoreeol = false, const unsigned char version = 2, bool debug = false);


     ~Pattern();


     Pattern(int size) {
         //pattern consisting only of fixed skips
         data = new unsigned char[size+1];
         for (int i = 0; i < size; i++) data[i] = ClassDecoder::skipclass;
         data[size] = ClassDecoder::delimiterclass;
     }

     void write(std::ostream * out, const unsigned char * corpusstart = NULL) const; 

     const size_t n() const;


     const size_t bytesize() const; 
     
     const size_t size() const { return n(); } 


     const unsigned int skipcount() const;

     const PatternCategory category() const;


     const bool isskipgram() const { return category() == SKIPGRAM; }
     const bool isflexgram() const { return category() == FLEXGRAM; }
     const bool unknown() const;
     
     Pattern operator [](int index) const { return Pattern(*this, index,1); } 

     const size_t hash() const;

     std::string tostring(const ClassDecoder& classdecoder) const; //pattern to string (decode)

     std::string decode(const ClassDecoder& classdecoder) const { return tostring(classdecoder); } //alias

     bool out() const;

     std::vector<unsigned int> tovector() const;

     bool operator==(const Pattern & other) const;
     bool operator!=(const Pattern & other) const;

     bool operator==(const PatternPointer & other) const; 
     bool operator!=(const PatternPointer & other) const; 

     void operator =(const Pattern & other);   


     bool operator<(const Pattern & other) const;
     bool operator>(const Pattern & other) const;

     Pattern operator +(const Pattern&) const;

     PatternPointer getpointer() const;

     int find(const Pattern & subpattern) const; 

     bool contains(const Pattern & subpattern) const; //does the pattern contain the smaller pattern?
   
     bool instanceof(const Pattern & skipgram) const; 
    

     int ngrams(std::vector<Pattern> & container, const int n) const; 
     int ngrams(std::vector<PatternPointer> & container, const int n) const; 

     int subngrams(std::vector<Pattern> & container, int minn = 1, int maxn=99) const; //return all subsumed ngrams (variable n)
     int subngrams(std::vector<PatternPointer> & container, int minn = 1, int maxn=99) const; //return all subsumed ngrams (variable n)

     int ngrams(std::vector<std::pair<Pattern,int>> & container, const int n) const; //return multiple ngrams
     int ngrams(std::vector<std::pair<PatternPointer,int>> & container, const int n) const; //return multiple ngrams

     int subngrams(std::vector<std::pair<Pattern,int>> & container, int minn = 1, int maxn=9) const; //return all subsumed ngrams (variable n)
     int subngrams(std::vector<std::pair<PatternPointer,int>> & container, int minn = 1, int maxn=9) const; //return all subsumed ngrams (variable n)

     int parts(std::vector<Pattern> & container) const; 
     int parts(std::vector<PatternPointer> & container) const; 
     
     int parts(std::vector<std::pair<int,int> > & container) const; 

     int gaps(std::vector<std::pair<int,int> > & container) const; 


     Pattern extractskipcontent(const Pattern & instance) const; 

     Pattern replace(int begin, int length, const Pattern & replacement) const;

     Pattern addskip(const std::pair<int,int> & gap) const;
     Pattern addskips(const std::vector<std::pair<int,int> > & gaps) const;
     Pattern addflexgaps(const std::vector<std::pair<int,int> > & gaps) const;

     Pattern reverse() const; 

     Pattern toflexgram() const;



     bool isgap(int i) const; 

     
     //NOT IMPLEMENTED YET:

     Pattern addcontext(const Pattern & leftcontext, const Pattern & rightcontext) const;    
     void mask(std::vector<bool> & container) const; //returns a boolean mask of the skipgram (0 = gap(encapsulation) , 1 = skipgram coverage)

     //sets an entirely new value
     void set(const unsigned char* dataref, const int size); 
};


Pattern patternfromfile(const std::string & filename); //helper function to read pattern from file, mostly for Cython

class PatternPointer {
    public:
     const static int patterntype = PATTERNPOINTER;
     unsigned char * data; 
     uint32_t bytes; //number of bytes
     uint32_t mask; //0 == NGRAM
                    //first bit high = flexgram, right-aligned, 0 = gap
                    //first bit low = skipgram, right-aligned, 0 = gap , max skipgram length 31 tokens
    
     PatternPointer() {
        data = NULL;
        bytes = 0;
        mask = 0;
     }

     PatternPointer(unsigned char* dataref, const unsigned int bytesize) {
         data = dataref;
         if (bytesize > B32) {
             std::cerr << "ERROR: Pattern too long for pattern pointer [" << bytesize << " bytes,explicit]" << std::endl;
             throw InternalError();
         }
         bytes = bytesize;
         mask = computemask();
     }

     PatternPointer(const Pattern & ref) {
         data = ref.data;
         const size_t b = ref.bytesize();
         if (b > B32) {
             std::cerr << "ERROR: Pattern too long for pattern pointer [" << b << " bytes,implicit]" << std::endl;
             throw InternalError();
         }
         bytes = b;
         mask = computemask();
     }
     PatternPointer(const Pattern * ref) {
         data = ref->data;
         const size_t b = ref->bytesize();
         if (b > B32) {
             std::cerr << "ERROR: Pattern too long for pattern pointer [" << b << " bytes,implicit]" << std::endl;
             throw InternalError();
         }
         bytes = b;
         mask = computemask();
     }
     PatternPointer(const PatternPointer& ref) {
         data = ref.data;
         bytes = ref.bytes;
         mask = ref.mask;
     }
     PatternPointer(const PatternPointer* ref) {
        data = ref->data;
        bytes = ref->bytes;
        mask = ref->mask;
     }
     PatternPointer & operator =(const PatternPointer & other) {
         data = other.data;
         bytes = other.bytes;
         mask = other.mask;
         // by convention, always return *this (for chaining)
         return *this;
     }
     PatternPointer(std::istream * in, bool ignoreeol = false, const unsigned char version = 2, unsigned char * corpusstart = NULL, bool debug = false); 

     void write(std::ostream * out, const unsigned char * corpusstart = NULL) const; 

     //slice construtors:
     PatternPointer(unsigned char *, unsigned int,unsigned int);
     PatternPointer(const PatternPointer&, unsigned int,unsigned int);
     PatternPointer(const Pattern&, unsigned int,unsigned int);

     uint32_t computemask() const;

     const size_t n() const;
     const size_t bytesize() const { return bytes; }
     const size_t size() const { return n(); }

     const size_t hash() const;
     
     const PatternCategory category() const;
     const bool isskipgram() const { return category() == SKIPGRAM; }
     const bool isflexgram() const { return category() == FLEXGRAM; }

     std::string tostring(const ClassDecoder& classdecoder) const; //pattern to string (decode)
     std::string decode(const ClassDecoder& classdecoder) const { return tostring(classdecoder); } //pattern to string (decode)
     bool out() const;
     
     bool operator==(const PatternPointer & other) const;
     bool operator!=(const PatternPointer & other) const { return !(*this == other); }

     bool operator==(const Pattern & other) const;
     bool operator!=(const Pattern & other) const { return !(*this == other); }

     PatternPointer toflexgram() const;
     bool isgap(int i) const; 


     PatternPointer& operator++();

     bool operator<(const PatternPointer & other) const {
         if (data == other.data) {
             if (bytes == other.bytes) {
                 return mask < other.mask;
             } else {
                 return bytes < other.bytes;
             }
         } else {
            return data < other.data;
        }
     }

     int ngrams(std::vector<PatternPointer> & container, const int n) const; 
     int subngrams(std::vector<PatternPointer> & container, int minn = 1, int maxn=9) const; //return all subsumed ngrams (variable n)
     int ngrams(std::vector<std::pair<PatternPointer,int>> & container, const int n) const; //return multiple ngrams
     int subngrams(std::vector<std::pair<PatternPointer,int>> & container, int minn = 1, int maxn=9) const; //return all subsumed ngrams (variable n)

     int parts(std::vector<PatternPointer> & container) const; 
     
     int parts(std::vector<std::pair<int,int> > & container) const; 

     int gaps(std::vector<std::pair<int,int> > & container) const; 

     const unsigned int skipcount() const;

     PatternPointer addskip(const std::pair<int,int> & gap) const;

     PatternPointer addskips(const std::vector<std::pair<int,int> > & gaps) const;
    
     int flexcollapse(unsigned char * collapseddata) const;

     bool instanceof(const Pattern & skipgram) const; 

     operator Pattern() { return Pattern(*this); } //cast overload
     Pattern pattern() const { return Pattern(*this); } //cast overload
};


static const unsigned char * tmp_unk = (const unsigned char *) "\2";
static const unsigned char * tmp_skipmarker = (const unsigned char *) "\3";
static const unsigned char * tmp_flexmarker = (const unsigned char *) "\4";
const Pattern SKIPPATTERN = Pattern((const unsigned char *) tmp_skipmarker,1);
const Pattern FLEXPATTERN = Pattern((const unsigned char*) tmp_flexmarker,1);
static const Pattern UNKPATTERN = Pattern((const unsigned char* ) tmp_unk,1);

namespace std {

    template <>
    struct hash<Pattern> {
     public: 
          size_t operator()(const Pattern &pattern) const throw() {                            
              return pattern.hash();              
          }
    };



    template <>
    struct hash<const Pattern *> {
     public: 
          size_t operator()(const Pattern * pattern) const throw() {                            
              return pattern->hash();              
          }
    };

    template <>
    struct hash<PatternPointer> {
     public: 
          size_t operator()(const PatternPointer &pattern) const throw() {                            
              return pattern.hash();              
          }
    };



    template <>
    struct hash<const PatternPointer *> {
     public: 
          size_t operator()(const PatternPointer * pattern) const throw() {                            
              return pattern->hash();              
          }
    };

}

#endif