Open Source Projects Europe

    int           m_maxWordLength;

    int           m_maxWordLength;

    // Current span. Might be jf.dockes@wanadoo.f

    // Current span. Might be jf.dockes@wanadoo.f

    std::string        m_span; 

    std::string        m_span; 

    std::vector <std::pair<int, int> > m_words_in_span;

    // Current word: no punctuation at all in there. Byte offset

    // Current word: no punctuation at all in there. Byte offset

    // relative to the current span and byte length

    // relative to the current span and byte length

    int           m_wordStart;

    int           m_wordStart;

    unsigned int  m_wordLen;

    unsigned int  m_wordLen;

    int           m_spanpos;

    int           m_spanpos;

    // It may happen that our cleanup would result in emitting the

    // It may happen that our cleanup would result in emitting the

    // same term twice. We try to avoid this

    // same term twice. We try to avoid this

    int           m_prevpos;

    int           m_prevpos;

#ifdef TEXTSPLIT_STATS

#ifdef TEXTSPLIT_STATS

    // Stats counters. These are processed in TextSplit rather than by a 

    // Stats counters. These are processed in TextSplit rather than by a 

    // TermProc so that we can take very long words (not emitted) into

    // TermProc so that we can take very long words (not emitted) into

    // account.

    // account.

    unsigned int  m_wordChars;

    unsigned int  m_wordChars;

    // This processes cjk text:

    // This processes cjk text:

    bool cjk_to_words(Utf8Iter *it, unsigned int *cp);

    bool cjk_to_words(Utf8Iter *it, unsigned int *cp);

    void discardspan();

    void discardspan();

    bool span_is_acronym(std::string *acronym);

    bool span_is_acronym(std::string *acronym);

};

};

#endif /* _TEXTSPLIT_H_INCLUDED_ */

#endif /* _TEXTSPLIT_H_INCLUDED_ */