【python网络爬虫案例】python爬取51Job招聘信息
# -*-coding:utf-8-*-
import requests
import pandas as pd
from pprint import pprint
from lxml import etree
import time
import warnings
import json
warnings.filterwarnings("ignore")"""
这个文件是最完整的代码，我们需要进行翻页操作，进行所有数据的爬取。
这里的翻页操作很简单，观察下面的url，不同页面只有一个地方不同。
https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE,2,1.html?
https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE,2,2.html?
https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE,2,3.html?
"""for i in range(1, 1501):
print("正在爬取第" + str(i) + "页的数据")
url_pre = "https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE,2,"
url_end = ".html?"
url = url_pre + str(i) + url_end
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0;
Win64;
x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}
web = requests.get(url, headers=headers)
web.encoding = "gbk"
dom = etree.HTML(web.text)
# 1、岗位名称
job_name = dom.xpath('//div[@class="dw_table"]/div[@class="el"]//p/span/a[@target="_blank"]/@title')
# 2、公司名称
company_name = dom.xpath('//div[@class="dw_table"]/div[@class="el"]/span[@class="t2"]/a[@target="_blank"]/@title')
# 3、工作地点
address = dom.xpath('//div[@class="dw_table"]/div[@class="el"]/span[@class="t3"]/text()')
# 4、工资
salary_mid = dom.xpath('//div[@class="dw_table"]/div[@class="el"]/span[@class="t4"]')
salary = [i.text for i in salary_mid]
# 5、发布日期
release_time = dom.xpath('//div[@class="dw_table"]/div[@class="el"]/span[@class="t5"]/text()')
# 6、获取二级网址url
deep_url = dom.xpath('//div[@class="dw_table"]/div[@class="el"]//p/span/a[@target="_blank"]/@href')
RandomAll = []
JobDescribe = []
CompanyType = []
CompanySize = []
Industry = []
# range(len(deep_url))
for i in range(len(deep_url)):
web_test = requests.get(deep_url[i], headers=headers)
web_test.encoding = "gbk"
dom_test = etree.HTML(web_test.text)
# 7、爬取经验、学历信息，先合在一个字段里面，以后再做数据清洗。命名为random_all
random_all = dom_test.xpath('//div[@class="tHeader tHjob"]//div[@class="cn"]/p[@class="msg ltype"]/text()')
# 8、岗位描述性息
job_describe = dom_test.xpath('//div[@class="tBorderTop_box"]//div[@class="bmsg job_msg inbox"]/p/text()')
# 9、公司类型
company_type = dom_test.xpath('//div[@class="tCompany_sidebar"]//div[@class="com_tag"]/p[1]/@title')
# 10、公司规模(人数)
company_size = dom_test.xpath('//div[@class="tCompany_sidebar"]//div[@class="com_tag"]/p[2]/@title')
# 11、所属行业(公司)
industry = dom_test.xpath('//div[@class="tCompany_sidebar"]//div[@class="com_tag"]/p[3]/@title')
# 将上述信息保存到各自的列表中
# print random_all
# aa = json.dumps(random_all, encoding='gbk', ensure_ascii=False)
aa = ["".join(item.split()) for item in random_all]
RandomAll.append(aa)
# job_describe = json.dumps(job_describe, encoding='gbk', ensure_ascii=False)
bb = ["".join(item.split()) for item in job_describe]
JobDescribe.append(bb)
# company_type = json.dumps(company_type, encoding='gbk', ensure_ascii=False)
CompanyType.append(company_type)
# company_size = json.dumps(company_size, encoding='gbk', ensure_ascii=False)
CompanySize.append(company_size)
# industry = json.dumps(industry, encoding='gbk', ensure_ascii=False)
Industry.append(industry)
# 为了反爬，设置睡眠时间
time.sleep(1)
# 由于我们需要爬取很多页，为了防止最后一次性保存所有数据出现的错误，因此，我们每获取一夜的数据，就进行一次数据存取。
df = pd.DataFrame()
df["岗位名称"] = job_name
df["公司名称"] = company_name
df["工作地点"] = address
df["工资"] = salary
df["发布日期"] = release_time
df["经验、学历"] = RandomAll
df["公司类型"] = CompanyType
df["公司规模"] = CompanySize
df["所属行业"] = Industry
df["岗位描述"] = JobDescribe
# 这里在写出过程中，有可能会写入失败，为了解决这个问题，我们使用异常处理。
try:
df.to_csv("job_info.csv", mode="ab+", header=None, index=None, encoding="gbk")
except Exception as e:
print("当页数据写入失败", str(e))
time.sleep(1)
print("数据爬取完毕，是不是很开心！！！")
参考：用 Python 做一个动态可视化的交互大屏
