diff -r 7c396c12d0f7 -r 36e090a48613 src/tokenizer_zhcn.cpp
--- a/src/tokenizer_zhcn.cpp	Sun Apr 20 10:43:03 2008 +0800
+++ b/src/tokenizer_zhcn.cpp	Sun Apr 20 22:34:22 2008 +0800
@@ -143,6 +143,10 @@ BYTE * CSphTokenizer_zh_CN_UTF8::GetToke
 	//to Lower
 	BYTE* tok_ptr = (BYTE*)tok;
 	BYTE* tok_max = tok_ptr + symlen;
+	m_pTokenStart = tok_ptr;
+	m_pTokenEnd = tok_max;
+	m_bTokenBoundary = false;
+
 	while ( tok_ptr<tok_max )
 	{
 		int iCode = sphUTF8Decode ( tok_ptr );
@@ -171,18 +175,23 @@ BYTE * CSphTokenizer_zh_CN_UTF8::GetToke
 			if(tag == 'w' || iCode == ' ')
 			{
 				//skip the 1st char
+				m_bBoundary = true;
 			}
 			else
 			{
+				m_bBoundary = false;
 				if ( m_iAccum == 0 )
 					m_pTokenStart = (BYTE*)tok;
 
 				AccumCodepoint ( iCode );
 			}
 			//AccumCodepoint ( iCode ); //will cut token is larger than SPH_MAX_WORD_LEN
-		}else
+		}else {
 			break;
+		}
 	}
+	if(m_bBoundary)
+		m_bTokenBoundary = true;
 	//?? m_pCur =  tok_ptr;
 	FlushAccum ();
 
@@ -193,10 +202,12 @@ BYTE * CSphTokenizer_zh_CN_UTF8::GetToke
 		//m_sAccum[0] = 0;
 		return GetToken();
 	}
-	//printf("%s,",m_sAccum);
-	if(symlen)
+	//printf("%s,",m_sA^ccum);
+	if(symlen){
+		//m_pTokenStart = m_sAccum;
+		//m_pTokenEnd = &m_sAccum[symlen];
 		return m_sAccum;
-	else{
+	}else{
 		//printf("\n");
 		return NULL;
 	}