Python数据分析之Python和Selenium爬取BOSS直聘岗位

作者：Andy_2259 时间：2021-04-27 08:53:19　

一、数据爬取的代码

#encoding='utf-8'
from selenium import webdriver
import time
import re
import pandas as pd
import os

def close_windows():
#如果有登录弹窗，就关闭
try:
time.sleep(0.5)
if dr.find_element_by_class_name("jconfirm").find_element_by_class_name("closeIcon"):
dr.find_element_by_class_name("jconfirm").find_element_by_class_name("closeIcon").click()
except BaseException as e:
print('close_windows,没有弹窗',e)

def get_current_region_job(k_index):
flag = 0
# page_num_set=0#每区获取多少条数据，对30取整

df_empty = pd.DataFrame(columns=['岗位', '地点', '薪资', '工作经验', '学历', '公司', '技能'])
while (flag == 0):
# while (page_num_set<151)&(flag == 0):#每次只能获取150条信息
time.sleep(0.5)
close_windows()
job_list = dr.find_elements_by_class_name("job-primary")
for job in job_list:#获取当前页的职位30条
job_name = job.find_element_by_class_name("job-name").text
# print(job_name)
job_area = job.find_element_by_class_name("job-area").text
salary = job.find_element_by_class_name("red").get_attribute("textContent") # 获取薪资
# salary_raw = job.find_element_by_class_name("red").get_attribute("textContent") # 获取薪资
# salary_split = salary_raw.split('·') # 根据·分割
# salary = salary_split[0] # 只取薪资，去掉多少薪

# if re.search(r'天', salary):
# continue

experience_education = job.find_element_by_class_name("job-limit").find_element_by_tag_name(
"p").get_attribute("innerHTML")

# experience_education_raw = '1-3年<em class="vline"></em>本科'
experience_education_raw = experience_education
split_str = re.search(r'[a-zA-Z =<>/"]{23}', experience_education_raw) # 搜索分割字符串<em class="vline"></em>
# print(split_str)

experience_education_replace = re.sub(r'[a-zA-Z =<>/"]{23}', ",", experience_education_raw) # 分割字符串替换为逗号
# print(experience_education_replace)

experience_education_list = experience_education_replace.split(',') # 根据逗号分割
# print('experience_education_list:',experience_education_list)

if len(experience_education_list)!=2:
print('experience_education_list不是2个，跳过该数据',experience_education_list)
break
experience = experience_education_list[0]
education = experience_education_list[1]
# print(experience)
# print(education)

company = job.find_element_by_class_name("company-text").find_element_by_class_name("name").text

skill_list = job.find_element_by_class_name("tags").find_elements_by_class_name("tag-item")
skill = []
for skill_i in skill_list:
skill_i_text = skill_i.text
if len(skill_i_text) == 0:
continue
skill.append(skill_i_text)
# print(job_name)
# print(skill)

df_empty.loc[k_index, :] = [job_name, job_area, salary, experience, education, company, skill]
k_index = k_index + 1
# page_num_set=page_num_set+1
print("已经读取数据{}条".format(k_index))

close_windows()
try:#点击下一页
cur_page_num=dr.find_element_by_class_name("page").find_element_by_class_name("cur").text
# print('cur_page_num',cur_page_num)

#点击下一页
element = dr.find_element_by_class_name("page").find_element_by_class_name("next")
dr.execute_script("arguments[0].click();", element)
time.sleep(1)
# print('点击下一页')

new_page_num=dr.find_element_by_class_name("page").find_element_by_class_name("cur").text
# print('new_page_num',new_page_num)

if cur_page_num==new_page_num:
flag = 1
break

except BaseException as e:
print('点击下一页错误',e)
break

print(df_empty)
if os.path.exists("数据.csv"):#存在追加，不存在创建
df_empty.to_csv('数据.csv', mode='a', header=False, index=None, encoding='gb18030')
else:
df_empty.to_csv("数据.csv", index=False, encoding='gb18030')

return k_index

def main():
# 打开浏览器
# dr = webdriver.Firefox()
global dr
dr = webdriver.Chrome()
# dr = webdriver.Ie()

# # 后台打开浏览器
# option=webdriver.ChromeOptions()
# option.add_argument('headless')
# dr = webdriver.Chrome(chrome_options=option)
# print("打开浏览器")

# 将浏览器最大化显示
dr.maximize_window()

# 转到目标网址
# dr.get("https://www.zhipin.com/job_detail/?query=Python&city=100010000&industry=&position=")#全国
dr.get("https://www.zhipin.com/c101010100/?query=Python&ka=sel-city-101010100")#北京
print("打开网址")
time.sleep(5)

k_index = 0#数据条数、DataFrame索引

flag_hot_city=0

for i in range(3,17,1):
# print('第',i-2,'页')

# try:

# 获取城市
close_windows()
hot_city_list = dr.find_element_by_class_name("condition-city").find_elements_by_tag_name("a")
close_windows()
# hot_city_list[i].click()#防止弹窗，改为下面两句
# element_hot_city_list_first = hot_city_list[i]
dr.execute_script("arguments[0].click();", hot_city_list[i])

# 输出城市名
close_windows()
hot_city_list = dr.find_element_by_class_name("condition-city").find_elements_by_tag_name("a")
print('城市：{}'.format(i-2),hot_city_list[i].text)
time.sleep(0.5)

# 获取区县
for j in range(1,50,1):
# print('第', j , '个区域')
# try:

# close_windows()
# hot_city_list = dr.find_element_by_class_name("condition-city").find_elements_by_tag_name("a")

# 在这个for循环点一下城市，不然识别不到当前页面已经更新了
close_windows()
hot_city_list = dr.find_element_by_class_name("condition-city").find_elements_by_tag_name("a")
close_windows()
# hot_city_list[i].click()#防止弹窗，改为下面
dr.execute_script("arguments[0].click();", hot_city_list[i])

#输出区县名称
close_windows()
city_district = dr.find_element_by_class_name("condition-district").find_elements_by_tag_name("a")
if len(city_district)==j:
print('遍历完所有区县，没有不可点击的，跳转下一个城市')
break
print('区县：',j, city_district[j].text)
# city_district_value=city_district[j].text#当前页面的区县值

# 点击区县
close_windows()
city_district= dr.find_element_by_class_name("condition-district").find_elements_by_tag_name("a")
close_windows()
# city_district[j].click()]#防止弹窗，改为下面两句
# element_city_district = city_district[j]
dr.execute_script("arguments[0].click();", city_district[j])

#判断区县是不是点完了
close_windows()
hot_city_list = dr.find_element_by_class_name("condition-city").find_elements_by_tag_name("a")
print('点击后这里应该是区县', hot_city_list[1].text)#如果是不限，说明点完了，跳出

hot_city_list = dr.find_element_by_class_name("condition-city").find_elements_by_tag_name("a")
print('如果点完了，这里应该是不限：',hot_city_list[1].text)

hot_city_list = dr.find_element_by_class_name("condition-city").find_elements_by_tag_name("a")
if hot_city_list[1].text == '不限':
print('当前区县已经点完了，点击下一个城市')
flag_hot_city=1
break

close_windows()
k_index = get_current_region_job(k_index)#获取职位，爬取数据

# 重新点回城市页面，再次获取区县。但此时多了区县，所以i+1
close_windows()
hot_city_list = dr.find_element_by_class_name("condition-city").find_elements_by_tag_name("a")
close_windows()
# hot_city_list[i+1].click()#防止弹窗，改为下面两句
# element_hot_city_list_again = hot_city_list[i+1]
dr.execute_script("arguments[0].click();", hot_city_list[i+1])

# except BaseException as e:
# print('main的j循环-获取区县发生错误:', e)
# close_windows()

time.sleep(0.5)

# except BaseException as e:
# print('main的i循环发生错误:',e)
# close_windows()

time.sleep(0.5)

# 退出浏览器
dr.quit()
# p1.close()

if __name__ == '__main__':
main()

二、获取到的数据如图所示

三、数据分析的代码

# coding=utf-8
import collections
import wordcloud
import re
import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt

plt.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文标签
plt.rcParams['axes.unicode_minus'] = False # 设置正常显示符号

def create_dir_not_exist(path): # 判断文件夹是否存在,不存在-新建
if not os.path.exists(path):
os.mkdir(path)

create_dir_not_exist(r'./image')
create_dir_not_exist(r'./image/city')

data = pd.read_csv('数据.csv', encoding='gb18030')
data_df = pd.DataFrame(data)
print("\n查看是否有缺失值\n", data_df.isnull().sum())

data_df_del_empty = data_df.dropna(subset=['岗位'], axis=0)
# print("\n删除缺失值‘岗位'的整行\n",data_df_del_empty)
data_df_del_empty = data_df_del_empty.dropna(subset=['公司'], axis=0)
# print("\n删除缺失值‘公司'的整行\n",data_df_del_empty)

print("\n查看是否有缺失值\n", data_df_del_empty.isnull().sum())
print('去除缺失值后\n', data_df_del_empty)

data_df_python_keyword = data_df_del_empty.loc[data_df_del_empty['岗位'].str.contains('Python|python')]
# print(data_df_python_keyword)#筛选带有python的行

# 区间最小薪资
data_df_python_keyword_salary = data_df_python_keyword['薪资'].str.split('-', expand=True)[0]
print(data_df_python_keyword_salary) # 区间最小薪资
# Dataframe新增一列在第列新增一列名为' ' 的一列数据
data_df_python_keyword.insert(7, '区间最小薪资(K)', data_df_python_keyword_salary)
print(data_df_python_keyword)

# 城市地区
data_df_python_keyword_location_city = data_df_python_keyword['地点'].str.split('·', expand=True)[0]
print(data_df_python_keyword_location_city) # 北京
data_df_python_keyword_location_district = data_df_python_keyword['地点'].str.split('·', expand=True)[1]
print(data_df_python_keyword_location_district) # 海淀区

data_df_python_keyword_location_city_district = []
for city, district in zip(data_df_python_keyword_location_city, data_df_python_keyword_location_district):
city_district = city + district
data_df_python_keyword_location_city_district.append(city_district)
print(data_df_python_keyword_location_city_district) # 北京海淀区
# Dataframe新增一列在第列新增一列名为' ' 的一列数据
data_df_python_keyword.insert(8, '城市地区', data_df_python_keyword_location_city_district)
print(data_df_python_keyword)

data_df_python_keyword.insert(9, '城市', data_df_python_keyword_location_city)
data_df_python_keyword.insert(10, '地区', data_df_python_keyword_location_district)
data_df_python_keyword.to_csv("data_df_python_keyword.csv", index=False, encoding='gb18030')

print('-------------------------------------------')

def draw_bar(row_lable, title):
figsize_x = 10
figsize_y = 6
global list1_education, list2_education, df1, df2
plt.figure(figsize=(figsize_x, figsize_y))
list1_education = []
list2_education = []
for df1, df2 in data_df_python_keyword.groupby(row_lable):
list1_education.append(df1)
list2_education.append(len(df2))
# print(list1_education)
# print(list2_education)
# 利用 * 解包方式将一个排序好的元组，通过元组生成器再转成list
# print(*sorted(zip(list2_education,list1_education)))
# print(sorted(zip(list2_education,list1_education)))
# 排序，两个列表对应原始排序,按第几个列表排序，注意先后位置
list2_education, list1_education = (list(t) for t in zip(*sorted(zip(list2_education, list1_education))))
plt.bar(list1_education, list2_education)
plt.title('{}'.format(title))
plt.savefig('./image/{}分析.jpg'.format(title))
# plt.show()
plt.close()

# 学历
draw_bar('学历', '学历')
draw_bar('工作经验', '工作经验')
draw_bar('区间最小薪资(K)', '14个热门城市的薪资分布情况(K)')
# -----------------------------------------
# 根据城市地区求均值
list_group_city1 = []
list_group_city2 = []

for df1, df2 in data_df_python_keyword.groupby(data_df_python_keyword['城市地区']):
# print(df1)
# print(df2)
list_group_city1.append(df1)
salary_list_district = [int(i) for i in (df2['区间最小薪资(K)'].values.tolist())]
district_salary_mean = round(np.mean(salary_list_district), 2) # 每个区县的平均薪资 round(a, 2)保留2位小数
list_group_city2.append(district_salary_mean)
list_group_city2, list_group_city1 = (list(t) for t in
zip(*sorted(zip(list_group_city2, list_group_city1), reverse=False)))
#
# print(list_group_city1)
# print(list_group_city2)

plt.figure(figsize=(10, 50))
plt.barh(list_group_city1, list_group_city2)
# 坐标轴上的文字说明
for ax, ay in zip(list_group_city1, list_group_city2):
# 设置文字说明第一、二个参数：坐标轴上的值；第三个参数：说明文字；ha:垂直对齐方式；va：水平对齐方式
plt.text(ay, ax, '％.2f' ％ ay, ha='center', va='bottom')
plt.title('14个热门城市的各区县招聘工资情况(K)')
plt.savefig('./image/14个热门城市的各区县招聘工资情况(K).jpg')
# plt.show()
plt.close()

# -----------------------------------------
# 根据城市分组排序，

list_group_city11 = []
list_group_city22 = []
list_group_city33 = []
list_group_city44 = []

for df_city1, df_city2 in data_df_python_keyword.groupby(data_df_python_keyword['城市']):
# print(df_city1)#市
# print(df_city2)
list_group_district2 = [] # 区县列表
district_mean_salary2 = [] # 工资均值列表
for df_district1, df_district2 in df_city2.groupby(data_df_python_keyword['地区']):
# print(df_district1)#区县
# print(df_district2)#工作
list_group_district2.append(df_district1) # 记录区县
salary_list_district2 = [int(i) for i in (df_district2['区间最小薪资(K)'].values.tolist())] # 工资列表
district_salary_mean2 = round(np.mean(salary_list_district2), 2) # 每个区县的平均薪资 round(a, 2)保留2位小数
district_mean_salary2.append(district_salary_mean2) # 记录区县的平均工作的列表

district_mean_salary2, list_group_district2 = (list(tt) for tt in zip(
*sorted(zip(district_mean_salary2, list_group_district2), reverse=True)))
plt.figure(figsize=(10, 6))
plt.bar(list_group_district2, district_mean_salary2)

# 坐标轴上的文字说明
for ax, ay in zip(list_group_district2, district_mean_salary2):
# 设置文字说明第一、二个参数：坐标轴上的值；第三个参数：说明文字；ha:垂直对齐方式；va：水平对齐方式
plt.text(ax, ay, '％.2f' ％ ay, ha='center', va='bottom')

plt.title('14个热门城市的各区县招聘工资情况_{}(K)'.format(df_city1))
plt.savefig('./image/city/14个热门城市的各区县招聘工资情况_{}(K).jpg'.format(df_city1))
# plt.show()
plt.close()

# ----------------------------------------------------

skill_all = data_df_python_keyword['技能']
print(skill_all)

skill_list = []

for i in skill_all:
# print(type(i))
print(i)
# print(i.split(", | ' | \[ | \] | \" | "))
result = re.split(r'[,\' \[, \] ]', i)
print(result)
# if type(i) == list:
skill_list = skill_list + result
print('++++++++++++++++++++++++++++++++')
# print(skill_list)

list_new = skill_list

# 词频统计
word_counts = collections.Counter(list_new) # 对分词做词频统计
word_counts_top10 = word_counts.most_common(30) # 获取前10最高频的词
# print (word_counts_top10) # 输出检查
# print (word_counts_top10[0][0]) # 输出检查

# 生成柱状图
list_x = []
list_y = []
for i in word_counts_top10:
list_x.append(i[0])
list_y.append(i[1])
print('list_x', list_x[1:])
print('list_y', list_y[1:])
plt.figure(figsize=(30, 5))
plt.bar(list_x[1:], list_y[1:])
plt.savefig('./image/技能栈_词频_柱状图.png')
# plt.show()
plt.close()

list_new = " ".join(list_new) # 列表转字符串，以空格间隔
# print(list_new)

wc = wordcloud.WordCloud(
width=800,
height=600,
background_color="#ffffff", # 设置背景颜色
max_words=50, # 词的最大数（默认为200）
max_font_size=60, # 最大字体尺寸
min_font_size=10, # 最小字体尺寸（默认为4）
# colormap='bone', # string or matplotlib colormap, default="viridis"
colormap='hsv', # string or matplotlib colormap, default="viridis"
random_state=20, # 设置有多少种随机生成状态，即有多少种配色方案
# mask=plt.imread("mask2.gif"), # 读取遮罩图片！！
font_path='simhei.ttf'
)
my_wordcloud = wc.generate(list_new)

plt.imshow(my_wordcloud)
plt.axis("off")
# plt.show()
wc.to_file('./image/技能栈_词云.png') # 保存图片文件
plt.close()