Python爬豆瓣电影实例

文件结构

html_downloader.py - 下载网页html内容

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import urllib2

class HtmlDownloader(object):

def downlod(self, url):
   if url is None:
     return None
   response = urllib2.urlopen(url)
   if response.getcode() != 200:
     return None
   return response.read()

html_outputer.py - 输出结果到文件中

#!/usr/bin/python
# -*- coding: UTF-8 -*-

class HtmlOutputer(object):

def collect_data(self, movie_data):
   if movie_data is None:
     return
   fout = open('output.html', 'a+')
   for data in movie_data:
     print data['name'] + '|', data['rate'] + '|', data['actor'], '\n'
     fout.write('％s,' ％ data['name'].encode('utf-8'))
     fout.write('％s,' ％ data['rate'])
     fout.write('％s\n' ％ data['actor'].encode('utf-8'))
   fout.close()

html_parser.py: 解析器：解析html的dom树

#!/usr/bin/python
# -*- coding: UTF-8 -*-
from bs4 import BeautifulSoup

class HtmlParser(object):

def __init__(self):
   pass

def parser_html(self, cnt):
   if cnt is None:
     return
   soup = BeautifulSoup(cnt, 'html.parser', from_encoding='utf-8')
   # movie_name, movie_desc, movie_rate =
   return self.get_movie_names(soup)

def get_movie_names(self, soup):
   movie_data = []
   movie_all = soup.find('div', class_='article').find_next('table').find_next_sibling('div').find_next_sibling('div').find_all('table')
   count = 1
   for movie_one in movie_all:
     movie_data.append(self.get_movie_name(movie_one))
     # if count > 2:
     #   break
     count += 1
   return movie_data

def get_movie_name(self, cnt):
   info = {}
   soup = BeautifulSoup(str(cnt), 'html.parser', from_encoding='utf-8')
   movie_one = soup.find('tr', class_='item').find_next('td').find_next_sibling('td').find('div', class_='pl2')
   info['name'] = movie_one.find('a').get_text().replace("\n", "").replace(" ", "")
   info['actor'] = movie_one.find('p', class_='pl').get_text().replace("\n", "").replace(" ", "")
   info['rate'] = movie_one.find('div', class_='star clearfix').find('span', class_='rating_nums').get_text()
   return info

spider_main.py - 主函数

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import html_parser, html_outputer, html_downloader

class SpiderMain(object):

def __init__(self):
   self.parser = html_parser.HtmlParser()
   self.outputer = html_outputer.HtmlOutputer()
   self.downloader = html_downloader.HtmlDownloader()

def craw(self, url):
   html_cnt = self.downloader.downlod(url)
   movie_data = self.parser.parser_html(html_cnt)
   self.outputer.collect_data(movie_data)

if __name__ == '__main__':
 url = 'https://movie.douban.com/tag/2017?start=100&type=T'
 spider = SpiderMain()
 spider.craw(url)

综述

其实就是使用了urllib2和BeautifulSoup库，没啥好说的，你也可以直接改url，然后更改html_parser.py文件来满足你自己的爬虫需求。当前也可以更改html_outputer.py来定义保存格式，目前是csv。