/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright (c) 2007 Sun Microsystems, Inc. All Rights Reserved.
 *
 * The contents of this file are subject to the terms of either the GNU Lesser
 * General Public License Version 2.1 only ("LGPL") or the Common Development and
 * Distribution License ("CDDL")(collectively, the "License"). You may not use this
 * file except in compliance with the License. You can obtain a copy of the CDDL at
 * http://www.opensource.org/licenses/cddl1.php and a copy of the LGPLv2.1 at
 * http://www.opensource.org/licenses/lgpl-license.php. See the License for the
 * specific language governing permissions and limitations under the License. When
 * distributing the software, include this License Header Notice in each file and
 * include the full text of the License in the License file as well as the
 * following notice:
 *
 * NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE
 * (CDDL)
 * For Covered Software in this distribution, this License shall be governed by the
 * laws of the State of California (excluding conflict-of-law provisions).
 * Any litigation relating to this License shall be subject to the jurisdiction of
 * the Federal Courts of the Northern District of California and the state courts
 * of the State of California, with venue lying in Santa Clara County, California.
 *
 * Contributor(s):
 *
 * If you wish your version of this file to be governed by only the CDDL or only
 * the LGPL Version 2.1, indicate your decision by adding "[Contributor]" elects to
 * include this software in this distribution under the [CDDL or LGPL Version 2.1]
 * license." If you don't indicate a single choice of license, a recipient has the
 * option to distribute your version of this file under either the CDDL or the LGPL
 * Version 2.1, or to extend the choice of license to its licensees as provided
 * above. However, if you add LGPL Version 2.1 code and therefore, elected the LGPL
 * Version 2 license, then the option applies only if the new code is made subject
 * to such option by the copyright holder.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#ifdef HAVE_ASSERT_H
#include <assert.h>
#endif

#include <algorithm>

#ifdef HAVE_ICONV_H
#include <iconv.h>
#endif

#include "pytrie_gen.h"

static const char*
skipSpace(const char* p)
{
    while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r')
        ++p;
    return p;
}

static const char*
skipNonSpace(const char* p)
{
    while (*p != '\0' && *p != ' ' && *p != '\t' && *p != '\n' && *p != '\r')
        ++p;
    return p;
}

static void
insertWordId(CPinyinTrieMaker::CWordSet& idset, CPinyinTrieMaker::TWordId id)
{
    CPinyinTrieMaker::CWordSet::iterator it = idset.find(id);
    if (it == idset.end())
        idset.insert(id);
    else {
        const CPinyinTrieMaker::TWordId& a = *it;
        if ((a.anony.m_bHide && !id.anony.m_bHide) || (a.anony.m_bHide == id.anony.m_bHide && a.anony.m_cost > id.anony.m_cost)) {
            idset.erase(it);
            idset.insert(id);
        }
    }
}

struct TSyllableInfo {
    std::string   m_py;
    int           m_cost;

    TSyllableInfo(const char* py=NULL, int cost=0) : m_py(py), m_cost(cost) {}
    bool operator< (const TSyllableInfo& b) const { return m_py < b.m_py; }
};

#ifdef HAVE_ICONV_H
bool isCorrectConverted(const char* utf8, iconv_t ic, iconv_t ric)
{
    static char gbstr[256];
    static char utstr[256];

    TIConvSrcPtr src = (TIConvSrcPtr)utf8;
    size_t srclen = strlen((char*)src)+1;
    char* dst = (char *)gbstr;
    size_t dstlen = 256;
    size_t res = iconv(ic, &src, &srclen, &dst, &dstlen);

    if (res != size_t(-1) && srclen == 0) {
        // do revert convertion and compare them
        src = (TIConvSrcPtr)gbstr;
        srclen = strlen((char*)src)+1;
        dst = (char *)utstr;
        dstlen = 256;
        res = iconv(ric, &src, &srclen, &dst, &dstlen);
        if (res != size_t(-1) && srclen == 0)
            return (strcmp(utf8, utstr) == 0);
    }
    return false;
}

//return: bit 0x1: contains some gbk out of gb2312, bit 0x2: contains some gb18030 outof gbk
unsigned getPureGBEncoding(const char* utf8str)
{
    static iconv_t ic_gb = iconv_open("GB2312", "UTF-8");
    static iconv_t ic_gbk = iconv_open("GBK", "UTF-8");
    static iconv_t ric_gb = iconv_open("UTF-8", "GB2312");
    static iconv_t ric_gbk = iconv_open("UTF-8", "GBK");

    unsigned ret = 0;

    if (!isCorrectConverted(utf8str, ic_gb, ric_gb)) {
        ret = 1; // at least it is contains some GBK char
        if (!isCorrectConverted(utf8str, ic_gbk, ric_gbk))
            ret = 3; //contains some GB18030-only char

        #ifdef DEBUG
            fprintf(stderr, "==> GB category of (%s) is (0x%x)\n ", utf8str, ret);
            fflush(stderr);
        #endif
    }
    return ret;
}
#else // !HAVE_ICONV_H
unsigned getPureGBEncoding(const char* utf8str)
{
    // FIXME
    return 0x3;
}
#endif // HAVE_ICONV_H

bool
parseLine(char* buf, char* word_buf, int& id, std::set<TSyllableInfo>& pyset)
{
    pyset.clear();

    /* ignore the empty lines and comment lines */
    if (*buf == '\n' || *buf == '#')
        return 0;

    char* word_start = word_buf;
    char* p = (char*)skipSpace(buf);
    char* t = (char*)skipNonSpace(p);
    while(p < t) *word_buf++ = *p++;
    *word_buf = 0;

    p = (char*)skipSpace(p);
    t = (char*)skipNonSpace(p);
    if (*t)
        *t++ = 0;
    id = atoi(p);
    p = (char*)skipSpace(t);
    while (*p) {
        const char* s = p;
        t = (char*)skipNonSpace(p);
        if (*t)
            *t++ = 0;
        while ((*p >= 'a' && *p <= 'z') || (*p == CPinyinTrie::SYLLABLE_BREAKER))
            ++p;
        if ((p > s) && ((*p == 0) || (*p == ':'))) {
            int  cost = 0;
            if (*p == ':') {
                *p++ = 0;
                cost = atoi(p);
            }
            pyset.insert(TSyllableInfo(s, cost));
        }
        p = (char*)skipSpace(t);
    }
    return pyset.size() > 0;
}


CPinyinTrieMaker::CPinyinTrieMaker()
    : m_RootNode(), m_FullSyllables(), m_StateMap(), m_AllNodes()
{
    m_AllNodes.push_back(&m_RootNode);
    m_RootNode.m_bExpanded = true;
    m_RootNode.m_PrimitiveNodes.insert(&m_RootNode);
    m_StateMap[&(m_RootNode.m_PrimitiveNodes)] = &m_RootNode;
}
/**********************************************************
    lexicon
        TAB(1)
         word id
        '
        4095;
**********************************************************/
#define RARE_MULTI_PHONETIC_STARTING_ID 140000 /* FIXME */
bool
CPinyinTrieMaker::constructFromLexicon(const char* fileName)
{
    static int  rmp_id = RARE_MULTI_PHONETIC_STARTING_ID;
    static char buf[4096];
    static char word_buf[2048];

    int id;
    bool suc = true;
    std::set<TSyllableInfo> pyset;
    FILE *fp = fopen(fileName, "r");
    printf("Adding pinyin and corresponding words..."); fflush(stdout);
    while (fgets(buf, 4096, fp) != NULL) {
        if (!parseLine(buf, word_buf, id, pyset)) {
            if (word_buf[0] != L'<' && word_buf[0] != 0) {
                if (m_Lexicon.size() < id+1) m_Lexicon.resize(id+1);
                m_Lexicon[id] = std::string(word_buf);
            }
            continue;
        }
        unsigned gbcategory = getPureGBEncoding(word_buf);

        std::set<TSyllableInfo>::iterator its = pyset.begin();
        std::set<TSyllableInfo>::iterator ite = pyset.end();
        for (; its != ite; ++its) {
            const char *t = its->m_py.c_str();
            int cost = its->m_cost;
            int myid = id;

            if (cost < 0) {
                cost = 30 / (-cost);
                myid = rmp_id ++;
            }

            if (m_Lexicon.size() < myid+1) m_Lexicon.resize(myid+1);
            m_Lexicon[myid] = std::string(word_buf);

            CPinyinTrieMaker::TWordId wid(myid, cost, its->m_cost < 0, gbcategory & 0x1, gbcategory & 0x2);
            suc = insertFullPinyinPair(t, wid) && suc;

            while (*t) {
                char *p = buf;
                while (*t != 0 && *t != CPinyinTrie::SYLLABLE_BREAKER)
                    *p++ = *t++;
                *p = 0;
                registerFullSyllable(buf);
                if (*t == CPinyinTrie::SYLLABLE_BREAKER)
                    ++t;
            }
        }
    }
    fclose(fp);

    std::string pyPrefix = "";

    printf("\n    %d primitive nodes", m_AllNodes.size());  fflush(stdout);

    /*
    printf("\n    Printing it to stderr...");
    print(stderr, &m_RootNode, pyPrefix);
    */

    printf("\nThreading non-complete pinyin...");  fflush(stdout);
    suc = threadNonCompletePinyin() && suc;
    printf("\n    %d total nodes", m_AllNodes.size());  fflush(stdout);

    /*
    printf("\n    Printing it to stderr...");
    fprintf(stderr, "\n\n\n\n-----------------------------\n\n\n\n");
    print(stderr, &m_RootNode, pyPrefix);
    */

    printf("\n");  fflush(stdout);

    return suc;
}

CPinyinTrieMaker::TNode::TNode()
    : m_bFullSyllableTransfer(false), m_bExpanded(false), m_WordIdSet(),
      m_Trans(), m_PrimitiveNodes(), m_SyllablePrefix()
{
}

bool
CPinyinTrieMaker::PNodeSet::operator< (const PNodeSet& another) const
{
    CNodeSet::const_iterator t1 = m_pns->begin();
    CNodeSet::const_iterator t2 = m_pns->end();
    CNodeSet::const_iterator a1 = another.m_pns->begin();
    CNodeSet::const_iterator a2 = another.m_pns->end();
    for (; t1 != t2 && a1 != a2; ++t1, ++a1) {
        if (*t1 < *a1) return true;
        if (*t1 > *a1) return false;
    }
    return (a1 != a2);
}

bool
CPinyinTrieMaker::PNodeSet::operator==(const PNodeSet& another) const
{
    CNodeSet::const_iterator t1 = m_pns->begin();
    CNodeSet::const_iterator t2 = m_pns->end();
    CNodeSet::const_iterator a1 = another.m_pns->begin();
    CNodeSet::const_iterator a2 = another.m_pns->end();
    for (; t1 != t2 && a1 != a2; ++t1, ++a1) {
        if (*t1 != *a1) return false;
    }
    return (a1 == a2 && t1 != t2);
}


void
CPinyinTrieMaker::print(FILE* fp, TNode* root, std::string& pinyin)
{
    if (root && root->m_WordIdSet.size() > 0) {
        fprintf(fp, "%s", pinyin.c_str());
        CWordSet::iterator itId = root->m_WordIdSet.begin();
        CWordSet::iterator itIdLast = root->m_WordIdSet.end();
        for (; itId != itIdLast; ++itId) {
            fprintf(fp, " %s", m_Lexicon[itId->anony.m_id].c_str());
        }
        fprintf(fp, "\n");
    }
    if (root) {
        CTrans::iterator itTrans = root->m_Trans.begin();
        CTrans::iterator itTransLast = root->m_Trans.end();
        for (; itTrans != itTransLast; ++itTrans) {
            pinyin += itTrans->first;
            print(fp, itTrans->second, pinyin);
            pinyin.resize(pinyin.size() -1);
        }
    }
}


/***********************************************************
    
    pinyin trie
***********************************************************/
bool
CPinyinTrieMaker::registerFullSyllable(const char* pinyin)
{
    CPinyinString str(pinyin);
    m_FullSyllables.insert(str);
    return true;
}


CPinyinTrieMaker::TNode*
CPinyinTrieMaker::insertTransfer(TNode* pnode, unsigned char c)
{
    CTrans::iterator itt = pnode->m_Trans.find(c);
    CTrans::iterator ite = pnode->m_Trans.end();
    if (itt == ite) {
        TNode *p = new TNode();
        m_AllNodes.push_back(p);
        pnode->m_Trans[c] = p;
        if (c != CPinyinTrie::SYLLABLE_BREAKER) {
            p->m_SyllablePrefix = pnode->m_SyllablePrefix;
            p->m_SyllablePrefix += c;
        }
        return p;
    }
    return itt->second;
}

/***********************************************************
    pinyin:   != NULL, 26'
               >0''
    wid   :   word id
***********************************************************/
bool
CPinyinTrieMaker::insertFullPinyinPair(const char* pinyin, TWordId wid)
{
    const char* p = pinyin;
    TNode *pnode = &m_RootNode;
    for (; *p; ++p) {
        if (*p == CPinyinTrie::SYLLABLE_BREAKER)
            pnode->m_bFullSyllableTransfer = true;
        pnode = insertTransfer(pnode, (unsigned char)*p);
        pnode->m_bExpanded = true;
        pnode->m_PrimitiveNodes.insert(pnode);
        m_StateMap[&(pnode->m_PrimitiveNodes)] = pnode;
    }
    if (*p-1 != CPinyinTrie::SYLLABLE_BREAKER) {
        pnode->m_bFullSyllableTransfer = true;
        pnode = insertTransfer(pnode, CPinyinTrie::SYLLABLE_BREAKER);
        pnode->m_bExpanded = true;
        pnode->m_PrimitiveNodes.insert(pnode);
        m_StateMap[&(pnode->m_PrimitiveNodes)] = pnode;
    }
    insertWordId(pnode->m_WordIdSet, wid);
    return true;
}

bool
CPinyinTrieMaker::threadNonCompletePinyin(void)
{
    CNodeList::iterator itNode = m_AllNodes.begin();
    for (; itNode != m_AllNodes.end(); ++itNode) {
        TNode* pnode = *itNode;
        if (!pnode->m_bExpanded)
            expandNode(pnode);

        if (pnode->m_SyllablePrefix.size() > 0 &&
                m_FullSyllables.find(pnode->m_SyllablePrefix) == m_FullSyllables.end() &&
                pnode->m_Trans.find(CPinyinTrie::SYLLABLE_BREAKER) == pnode->m_Trans.end()) {
            addNonCompleteSyllableTransfer(pnode);
        }
    }
    return true;
}

/**
* For those node which are added after fullComplete PINYIN string,
* give their succ nodes, for example, for "dian'ying dui'yu dian'hua", an
* new state "d'" would be expand to "d'y d'h"
*/
void
CPinyinTrieMaker::expandNode(TNode* pnode)
{
    std::map<char, CNodeSet> combTrans;

    CNodeSet::iterator itNode = pnode->m_PrimitiveNodes.begin();
    CNodeSet::iterator itNodeLast = pnode->m_PrimitiveNodes.end();
    for (; itNode != itNodeLast; ++itNode) {
         CTrans::iterator itTrans = (*itNode)->m_Trans.begin();
         CTrans::iterator itTransLast = (*itNode)->m_Trans.end();
         for (; itTrans != itTransLast; ++itTrans) {
             if (itTrans->first == CPinyinTrie::SYLLABLE_BREAKER &&
                        !(*itNode)->m_bFullSyllableTransfer) {
                 continue;
             }
             combTrans[itTrans->first].insert(itTrans->second);
         }
    }

    std::map<char, CNodeSet>::iterator itCombTrans = combTrans.begin();
    std::map<char, CNodeSet>::iterator itCombTransLast = combTrans.end();
    for (; itCombTrans != itCombTransLast; ++itCombTrans) {
        //if a new state, ie new node set appear, create the new state --> ps
        //esle let ps = the founded old state, let transfer(c) = state
        unsigned char c = itCombTrans->first;
//        if (c == CPinyinTrie::SYLLABLE_BREAKER && !pnode->m_bFullSyllableTransfer) {
//            continue;
//        }
        TNode* pChildNode = NULL;
        CStateMap::iterator itStateMap = m_StateMap.find(&itCombTrans->second);
        if (itStateMap != m_StateMap.end()) {
            pChildNode = itStateMap->second;
        } else {
            pChildNode = new TNode();
            m_AllNodes.push_back(pChildNode);
            pChildNode->m_PrimitiveNodes = itCombTrans->second;
            m_StateMap[&pChildNode->m_PrimitiveNodes] = pChildNode;
            if (c != CPinyinTrie::SYLLABLE_BREAKER) {
                pChildNode->m_SyllablePrefix = pnode->m_SyllablePrefix;
                pChildNode->m_SyllablePrefix += c;
            } else {
                CNodeSet::iterator itps = pChildNode->m_PrimitiveNodes.begin();
                CNodeSet::iterator itpse = pChildNode->m_PrimitiveNodes.end();
                for (; itps != itpse; ++itps) {
                    CWordSet::iterator ita = (*itps)->m_WordIdSet.begin();
                    CWordSet::iterator itb = (*itps)->m_WordIdSet.end();
                    for (; ita != itb; ++ita)
                        insertWordId(pChildNode->m_WordIdSet, *ita);
                    //pChildNode->m_WordIdSet.insert((*itps)->m_WordIdSet.begin(), (*itps)->m_WordIdSet.end());
                }
                pnode->m_bFullSyllableTransfer = false;
            }
        }
        pnode->m_Trans[c] = pChildNode;
    }
    pnode->m_bExpanded = true;
}

void
CPinyinTrieMaker::addNonCompleteSyllableTransfer(TNode* pnode)
{
    CNodeSet syChildren;
    TNode* pChildNode = NULL;

    findSyllableChildren(pnode, syChildren);
    if (syChildren.size() == 0)  //z, c, s with only zh, ch, sh children
        return;
    CStateMap::iterator itStateMap = m_StateMap.find(&syChildren);
    if (itStateMap != m_StateMap.end()) {
        pChildNode = itStateMap->second;
    } else {
        pChildNode = new TNode();
        m_AllNodes.push_back(pChildNode);
        pChildNode->m_PrimitiveNodes = syChildren;
        m_StateMap[&pChildNode->m_PrimitiveNodes] = pChildNode;
        CNodeSet::iterator itps = pChildNode->m_PrimitiveNodes.begin();
        CNodeSet::iterator itpse = pChildNode->m_PrimitiveNodes.end();
        for (; itps != itpse; ++itps) {
            CWordSet::iterator ita = (*itps)->m_WordIdSet.begin();
            CWordSet::iterator itb = (*itps)->m_WordIdSet.end();
            for (; ita != itb; ++ita)
                insertWordId(pChildNode->m_WordIdSet, *ita);
            //pChildNode->m_WordIdSet.insert((*itps)->m_WordIdSet.begin(), (*itps)->m_WordIdSet.end());
        }
    }
    pnode->m_Trans[CPinyinTrie::SYLLABLE_BREAKER] = pChildNode;
    // this is default: pnode->m_bFullSyllableTransfer = false;
}

int
CPinyinTrieMaker::findSyllableChildren(const TNode *pn, CNodeSet& children)
{
    CNodeSet::iterator itNode = pn->m_PrimitiveNodes.begin();
    CNodeSet::iterator itNodeLast = pn->m_PrimitiveNodes.end();
    for (children.clear(); itNode != itNodeLast; ++itNode) {
        findPrimitiveSyllableChildren(*itNode, children);
    }
    return children.size();
}

void
CPinyinTrieMaker::findPrimitiveSyllableChildren(const TNode *pn, CNodeSet& children)
{
    CTrans::const_iterator it = pn->m_Trans.begin();
    CTrans::const_iterator ite= pn->m_Trans.end();
    for (; it != ite; ++it) {
        if (it->first != CPinyinTrie::SYLLABLE_BREAKER) {
            if (it->first == 'h' &&
                (pn->m_SyllablePrefix == "c" || pn->m_SyllablePrefix == "z" ||
                 pn->m_SyllablePrefix == "s" ) ) {
                continue;
            }
            findPrimitiveSyllableChildren(it->second, children);
        } else {
            if (pn->m_bFullSyllableTransfer) {
                children.insert(it->second);
            }
        }
    }
}

bool
CPinyinTrieMaker::write(const char* fileName, CWordEvaluator* psrt)
{
    bool suc = false;
    FILE* fp = fopen(fileName, "wb");
    if (fp != NULL) {
        suc = write(fp, psrt);
        fclose(fp);
    }
    return suc;
}

bool
CPinyinTrieMaker::write(FILE *fp, CWordEvaluator* psrt)
{
    bool suc = true;
    static TWCHAR wbuf[1024];

    std::map<TNode*, unsigned int> nodeOffsetMap;

    /*the file started with m_nWord, the size itself do not included here*/
    unsigned int nWord = m_Lexicon.size();
    unsigned int nNode = m_AllNodes.size();
    unsigned int lexiconOffset;
    unsigned int offset = sizeof(unsigned int) * 3;

    CNodeList::const_iterator itNode = m_AllNodes.begin();
    CNodeList::const_iterator itNodeLast = m_AllNodes.end();
    for (; itNode != itNodeLast; ++itNode) {
        nodeOffsetMap[*itNode] = offset;
        offset += CPinyinTrie::TNode::size_for((*itNode)->m_Trans.size(),
                                               (*itNode)->m_WordIdSet.size());
    }
    lexiconOffset = offset;
    CLexicon::const_iterator itWordStr = m_Lexicon.begin();
    CLexicon::const_iterator itWordStrLast = m_Lexicon.end();
    for (; itWordStr != itWordStrLast; ++itWordStr) {
        MBSTOWCS(wbuf, itWordStr->c_str(), 1024);
        int sz = WCSLEN(wbuf);
        offset += (sz+1)*sizeof(TWCHAR);
    }

    suc = (fwrite(&offset, sizeof(unsigned int), 1, fp) == 1);
    suc = (fwrite(&nWord, sizeof(unsigned int), 1, fp) == 1);
    suc = (fwrite(&nNode, sizeof(unsigned int), 1, fp) == 1);
    suc = (fwrite(&lexiconOffset, sizeof(unsigned int), 1, fp) == 1);

    itNode = m_AllNodes.begin();
    itNodeLast = m_AllNodes.end();
    for (; itNode != itNodeLast && suc; ++itNode) {
        CPinyinTrie::TNode outNode;
        outNode.m_nTransfer = (*itNode)->m_Trans.size();
        outNode.m_nWordId = (*itNode)->m_WordIdSet.size();
        outNode.m_bFullSyllableTransfer = (*itNode)->m_bFullSyllableTransfer;

        //determine this node's GB category, have some pure gb2312 words, or all GBK/GB18030 words
        outNode.m_bGBK = 1;
        outNode.m_bGB18030 = 1;

        TNode* itequ = *itNode;
        if (outNode.m_nWordId == 0) {
            if ((*itNode)->m_Trans.find(CPinyinTrie::SYLLABLE_BREAKER) != (*itNode)->m_Trans.end()) {
                itequ = (*itNode)->m_Trans[CPinyinTrie::SYLLABLE_BREAKER];
                if (itequ->m_WordIdSet.size() == 0) {
                    outNode.m_bGBK = 0;
                    outNode.m_bGB18030 = 0;
                }
            } else {
                outNode.m_bGBK = 0;
                outNode.m_bGB18030 = 0;
            }
        }
        CWordSet::iterator itId = itequ->m_WordIdSet.begin();
        CWordSet::iterator itIdLast = itequ->m_WordIdSet.end();
        for (; itId != itIdLast && outNode.m_bGBK; ++itId) {
            outNode.m_bGB18030 &= itId->anony.m_bGB18030;
            outNode.m_bGBK &= itId->anony.m_bGBK;
        }
        #ifdef DEBUG
            if (outNode.m_bGBK) {
                CWordSet::iterator itId = (*itNode)->m_WordIdSet.begin();
                CWordSet::iterator itIdLast = (*itNode)->m_WordIdSet.end();
                fprintf(stderr, "========>(");
                for (; itId != itIdLast; ++itId) {
                    fprintf(stderr, " %d-%1d", itId->anony.m_id, itId->anony.m_bGBK);
                }
                fprintf(stderr, " )\n\n");
                fflush(stderr);
            }
        #endif
        suc = (fwrite(&outNode, sizeof(outNode), 1, fp) == 1);

        CTrans::iterator itTrans = (*itNode)->m_Trans.begin();
        CTrans::iterator itTransLast = (*itNode)->m_Trans.end();
        for (; itTrans != itTransLast && suc; ++itTrans) {
            CPinyinTrie::TTransUnit tru;
            tru.m_Char = itTrans->first;
            tru.m_Offset = nodeOffsetMap[itTrans->second];
            assert(tru.m_Offset != 0);
            suc = (fwrite(&tru, sizeof(tru), 1, fp) == 1);
        }

        CWordVec vec;
        itId = (*itNode)->m_WordIdSet.begin();
        itIdLast = (*itNode)->m_WordIdSet.end();
        for (; itId != itIdLast; ++itId)
            vec.push_back(TWordInfo(*itId, psrt->getCost(*itId), psrt->isSeen(*itId)));
        std::make_heap(vec.begin(), vec.end());
        std::sort_heap(vec.begin(), vec.end());

        CWordVec::iterator itv = vec.begin();
        CWordVec::iterator itve = vec.end();
        for (; itv != itve && suc; ++itv) {
            CPinyinTrie::TWordIdInfo wi;
            wi.m_id = itv->m_id.anony.m_id;
            wi.m_bGBK = itv->m_id.anony.m_bGBK;
            wi.m_bGB18030 = itv->m_id.anony.m_bGB18030;
            wi.m_len = m_Lexicon[itv->m_id.anony.m_id].size();
            wi.m_bSeen = ((itv->m_bSeen)?(1):(0));
            wi.m_cost = itv->m_id.anony.m_cost;
            suc = (fwrite(&wi, sizeof(wi), 1, fp) == 1);
        }
    }
    itWordStr = m_Lexicon.begin();
    itWordStrLast = m_Lexicon.end();
    for (; itWordStr != itWordStrLast && suc; ++itWordStr) {
        MBSTOWCS(wbuf, itWordStr->c_str(), 1024);
        int sz = WCSLEN(wbuf);
        suc = (fwrite(wbuf, (sz+1)*sizeof(TWCHAR), 1, fp) == 1);
    }
    return suc;
}

Indexes created Sat Nov 07 19:43:50 UTC 2009

Terms of Use | Privacy | Trademarks | Copyright Policy | Site Guidelines | Help
Your use of this web site or any of its content or software indicates your agreement to be bound by these Terms of Use.
Copyright © 1995-2008 Sun Microsystems, Inc.