clucene 读源码记录
0\ iconv 的使用http://www.gnu.org/software/libc/manual/html_node/iconv-Examples.html#iconv-Examples
1\ TestUtf8 中的 测试把GBK中的数据转换为unicode，然后索引
void _Index(CuTest *tc, IndexWriter* ndx,char* file){ char path[CL_MAX_PATH]; TCHAR tlang[20]; strcpy(path,clucene_data_location);strcat(path,"/utf8text");CuAssert(tc,_T("Utf8 directory does not exist"),Misc::dir_Exists(path));strcat(path,"/");strcat(path,file);/*strcat(path,"_utf8.txt");*/strcat(path,"_gbk.txt");CuAssert(tc,_T("Language file does not exist"),Misc::dir_Exists(path));STRCPY_AtoT(tlang,file,CL_MAX_PATH); Document doc;doc.add(*Field::Keyword(_T("language"),tlang)); jstreams::FileReader* fr = new jstreams::FileReader(path, "GBK"); // StandardAnalyzer analyzer; // TokenStream* stream = analyzer.tokenStream(NULL, _CLNEW CL_NS(util)::Reader(fr,true));doc.add(*Field::Text(_T("contents"), _CLNEW CL_NS(util)::Reader(fr,true) ));ndx->addDocument(&doc); }int32_t FileInputStream::fillBuffer(char* start, int32_t space) { if (file == 0) return -1; // read into the buffer int32_t nwritten = fread(start, 1, space, file); // check the file stream status if (ferror(file)) { error = "Could not read from file '" + filepath + "'."; fclose(file); file = 0; status = Error; return -1; } if (feof(file)) { fclose(file); file = 0; } return nwritten;}template <class T> template <class char>void InputStreamBuffer<T>::setSize(int32_t size) { // store pointer information int32_t offset = (int32_t)(readPos - start);// allocate memory in the buffer if ( start == 0 )start = (T*)malloc(size*sizeof(T));elsestart = (T*)realloc(start, size*sizeof(T)); this->size = size; // restore pointer information readPos = start + offset;} iconv_t converter = iconv_open("UCS-2-INTERNAL", "GBK");//iconv_t converter = iconv_open("UCS-2-INTERNAL", "UTF-8");// iconv_t converter = iconv_open("UCS-2-INTERNAL", "ASCII"); const char *inbuf ="我喜欢你欧阳"; size_t inbytesleft = strlen(inbuf); wchar_t* start=(wchar_t*)malloc(inbytesleft*sizeof(wchar_t)); memset(start,0,inbytesleft*sizeof(wchar_t)); size_t outbytesleft = sizeof(wchar_t)*inbytesleft; char *outbuf = (char*)start; size_t r = iconv(converter, &inbuf, &inbytesleft, &outbuf, &outbytesleft); iconv_close(converter); doc.add(*Field::Text(_T("contents"),start));//代码转换:从一种编码转为另一种编码int code_convert(char *from_charset,char *to_charset,const char *inbuf,size_t inlen,char *outbuf,size_t outlen){iconv_t cd;int rc;const char **pin = &inbuf;char **pout = &outbuf;cd = iconv_open(to_charset,from_charset);if (cd==0) return -1;memset(outbuf,0,outlen);if (iconv(cd,pin,&inlen,pout,&outlen)==-1) return -1;iconv_close(cd);return 0;}//UNICODE码转为GB2312码int u2g(char *inbuf,size_t inlen,char *outbuf,size_t outlen){return code_convert("utf-8","gb2312",inbuf,inlen,outbuf,outlen);}//GB2312码转为UNICODE码int g2u(char *inbuf,size_t inlen,char *outbuf,size_t outlen){return code_convert("gb2312","utf-8",inbuf,inlen,outbuf,outlen);} iconv_t converter = iconv_open("UCS-2-INTERNAL", "UTF-8"); //gb2312码转为unicode码 char *in_gb2312="我喜欢你欧阳"; size_t tmpleng = strlen(in_gb2312);// char out[500]; char* out=(char*)malloc(3*tmpleng*sizeof(char)); int rec = g2u(in_gb2312,strlen(in_gb2312),out,3*tmpleng); const char * inbuf=out; size_t inbytesleft = strlen(inbuf); wchar_t* start=(wchar_t*)malloc(inbytesleft*sizeof(wchar_t)); memset(start,0,inbytesleft*sizeof(wchar_t)); size_t outbytesleft = sizeof(wchar_t)*inbytesleft; char *outbuf = (char*)start; size_t r = iconv(converter, &inbuf, &inbytesleft, &outbuf, &outbytesleft); iconv_close(converter); doc.add(*Field::Text(_T("contents"),start));