python正向最大匹配分词和逆向最大匹配分词的实例

正向最大匹配

# -*- coding:utf-8 -*-

CODEC='utf-8'

def u(s, encoding):
 'converted other encoding to unicode encoding'
 if isinstance(s, unicode):
   return s
 else:
   return unicode(s, encoding)

def fwd_mm_seg(wordDict, maxLen, str):
 'forward max match segment'
 wordList = []
 segStr = str
 segStrLen = len(segStr)
 for word in wordDict:
   print 'word: ', word
 print "\n"
 while segStrLen > 0:
   if segStrLen > maxLen:
     wordLen = maxLen
   else:
     wordLen = segStrLen
   subStr = segStr[0:wordLen]
   print "subStr: ", subStr
   while wordLen > 1:
     if subStr in wordDict:
       print "subStr1: ％r" ％ subStr
       break
     else:
       print "subStr2: ％r" ％ subStr
       wordLen = wordLen - 1
       subStr = subStr[0:wordLen]
#      print "subStr3: ", subStr
   wordList.append(subStr)
   segStr = segStr[wordLen:]
   segStrLen = segStrLen - wordLen
 for wordstr in wordList:
   print "wordstr: ", wordstr
 return wordList

def main():
 fp_dict = open('words.dic')
 wordDict = {}
 for eachWord in fp_dict:
   wordDict[u(eachWord.strip(), 'utf-8')] = 1
 segStr = u'你好世界hello world'
 print segStr
 wordList = fwd_mm_seg(wordDict, 10, segStr)
 print "==".join(wordList)

if __name__ == '__main__':
 main()

逆向最大匹配

# -*- coding:utf-8 -*-

def u(s, encoding):
 'converted other encoding to unicode encoding'
 if isinstance(s, unicode):
   return s
 else:
   return unicode(s, encoding)

CODEC='utf-8'

def bwd_mm_seg(wordDict, maxLen, str):
 'forward max match segment'
 wordList = []
 segStr = str
 segStrLen = len(segStr)
 for word in wordDict:
   print 'word: ', word
 print "\n"
 while segStrLen > 0:
   if segStrLen > maxLen:
     wordLen = maxLen
   else:
     wordLen = segStrLen
   subStr = segStr[-wordLen:None]
   print "subStr: ", subStr
   while wordLen > 1:
     if subStr in wordDict:
       print "subStr1: ％r" ％ subStr
       break
     else:
       print "subStr2: ％r" ％ subStr
       wordLen = wordLen - 1
       subStr = subStr[-wordLen:None]
#      print "subStr3: ", subStr
   wordList.append(subStr)
   segStr = segStr[0: -wordLen]
   segStrLen = segStrLen - wordLen
 wordList.reverse()
 for wordstr in wordList:
   print "wordstr: ", wordstr
 return wordList

def main():
 fp_dict = open('words.dic')
 wordDict = {}
 for eachWord in fp_dict:
   wordDict[u(eachWord.strip(), 'utf-8')] = 1
 segStr = ur'你好世界hello world'
 print segStr
 wordList = bwd_mm_seg(wordDict, 10, segStr)
 print "==".join(wordList)

if __name__ == '__main__':
 main()

来源：https://blog.csdn.net/Yan456jie/article/details/78788783