python 高效去重复 支持GB级别大文件的示例代码

如下所示：

#coding=utf-8

import sys, re, os

def getDictList(dict):
 regx = '''[\w\~`\!\@\#\$\％\^\&\*\(\)\_\-\+\=\[\]\{\}\:\;\,\.\/\<\>\?]+'''
 with open(dict) as f:
   data = f.read()
   return re.findall(regx, data)

def rmdp(dictList):
 return list(set(dictList))

def fileSave(dictRmdp, out):
 with open(out, 'a') as f:
   for line in dictRmdp:
     f.write(line + '\n')

def main():
 try:
   dict = sys.argv[1].strip()
   out = sys.argv[2].strip()
 except Exception, e:
   print 'error:', e
   me = os.path.basename(__file__)
   print 'usage: ％s <input> <output>' ％me
   print 'example: ％s dict.txt dict_rmdp.txt' ％me
   exit()

dictList = getDictList(dict)
 dictRmdp = rmdp(dictList)
 fileSave(dictRmdp, out)

if __name__ == '__main__':
 main()

来源：https://blog.csdn.net/meinaozi/article/details/79326512