python去重,一个由dict组成的list的去重示例

背景：有一个list，里面的每一个元素都是dict，根据某一个key进行去重，在这里，key代表question

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# created by fhqplzj on 2017/12/07 上午11:38
from itertools import groupby
from operator import itemgetter

import pandas as pd

def distinct(items):
 questions = map(itemgetter('question'), items)
 df = pd.DataFrame({
   'items': items,
   'questions': questions
 })
 return df.drop_duplicates(['questions'])['items'].tolist()

def distinct2(items):
 exist_questions = set()
 result = []
 for item in items:
   question = item['question']
   if question not in exist_questions:
     exist_questions.add(question)
     result.append(item)
 return result

def distinct3(items):
 key = itemgetter('question')
 items = sorted(items, key=key)
 return [next(v) for _, v in groupby(items, key=key)]

def distinct4(items):
 from itertools import compress
 mask = (~pd.Series(map(itemgetter('question'), items)).duplicated()).tolist()
 return list(compress(items, mask))

if __name__ == '__main__':
 data = [
   {'question': 'a', 'ans': 'b'},
   {'question': 'b', 'ans': 'd'},
   {'question': 'a', 'ans': 'p'},
   {'question': 'b', 'ans': 'e'}
 ]
 print distinct4(data)

来源：https://blog.csdn.net/ASD991936157/article/details/78740610