如何通过网络爬虫获取网站相关数据？

import os 
import pandas as pd 
import requests
from lxml import etree

#可视化模块
import matplotlib.pyplot as plt

#可视化包可根据需求使用不同包如下
import seaborn as sns 
from pycharts.charts import *
import plotly.graph_objects as go
import plotly.express as px

#从address_msg中获取区-zone
df['zone'] = df['address_msg'].apply(lambda x:str(x)[0:int(str(x).find('区')+1)].replace('海口·',''))
#对zone中为空的补充为海口
df['zone'] = df['zone'].apply(lambda x:'海口' if x == '' else x)
df.head()

df['low_salary'] = df['salary_range'].apply(
    lambda i: str(i)[0:int(str(i).find('-'))]).astype(int)
df['high_slary'] = df['salary_range'].apply(
    lambda i: str(i)[int(str(i).find('-')+1):int(str(i).find('K'))]).astype(int)
df['mean_salary'] = df[['low_salary','high_slary']].apply('mean',axis = 1).astype(float)
#年薪
df['year_salary'] = df['salary_range'].apply(
    lambda i:str(i)[str(i).find('·')+1:str(i).find('薪')+1])
df['year_salary'] = df['year_salary'].apply(lambda x:'12薪' if x == '' else x)
#添加市辖区平均薪资
df['zone_salary'] = df.groupby('zone')['mean_salary'].transform('mean')
df['zone_salary'] = df['zone_salary'].apply(lambda i:'{:.2f}'.format(round(i,2)))
df.head()

zone_df = pd.DataFrame(df['zone'].value_counts()).reset_index().rename(
    columns={'index': 'zone', 'zone': 'counts'})
zone_df.loc[3,'zone'] = '海口'
fig = plt.figure(figsize=(10,8))
ax = fig.add_subplot(1,1,1)
patches,text,autotext = plt.pie(x = zone_df['counts'],#数据
        labels = zone_df['zone'], #标签
       autopct='%.2f%%', #显示百分比
       startangle = -25,
       explode=[0.02,0,0,0,0],
       textprops = {'fontsize':14},
       colors = ['rosybrown','lightblue','tan','orange','darkkhaki'],
       shadow = True
)
text[0].set_color('gray')
text[0].set_fontsize = 20
patches[0].set_hatch('|')
patches[0].set_alpha(0.3)
plt.legend(frameon = False,title = '辖区')
plt.title('海口辖区招聘数据分布')
plt.show()

zone_sal_df = df.groupby(['zone','year_salary'],as_index=False)[['job_name']].count()
plt.figure(figsize=(12,6))
sns.barplot(x = 'zone',y ='job_name',hue = 'year_salary',data = zone_sal_df)
#添加每个市辖区总的招聘数量
total_c = pd.DataFrame(zone_sal_df.groupby('zone')['job_name'].sum()).reset_index()
plt.plot(total_c['zone'],total_c['job_name'],color = 'gray',label = 'total_amount')
for i,j in zip(total_c['zone'],total_c['job_name']):
    plt.text(i,j+2,j)
plt.xlabel('市辖区')
plt.ylabel('招聘数量')
plt.title('海口市辖区招聘年薪分布')
plt.legend(loc = 'upper left',frameon = False)
plt.show()

fig,axes = plt.subplots(1,2,sharey = True,figsize = (9,8))
sns.violinplot(x = 'zone',y = 'low_salary',data = df,ax = axes[0])
sns.scatterplot(x = 'high_slary',y = 'low_salary',style = 'year_salary',hue = 'education',data = df,ax = axes[1])

zone_pro = df.groupby(['zone', 'profession_dir'], as_index=False)[
    ['job_name']].count().sort_values(by='job_name', ascending=False)
#行业招聘前10分布
print('行业前10招聘数量占比达：{:.2f}%'.format(round(zone_pro.groupby('profession_dir', as_index=False)[['job_name']].sum(
).sort_values(by='job_name', ascending=False).head(10)['job_name'].sum()
      /zone_pro['job_name'].sum(), 4)*100))
# zone_pro = df.groupby(['zone', 'profession_dir'], as_index=False)[
#     ['job_name']].count().sort_values(by='job_name', ascending=False)
# 饼图查看每个行业招聘数量占比
fig = plt.figure(figsize=(10, 10))
plt.pie(x=zone_pro.groupby('profession_dir',as_index=False)[['job_name']].sum().sort_values(by = 'job_name',ascending = False).head(10)['job_name'],
        labels=zone_pro.groupby('profession_dir',as_index=False)[['job_name']].sum().sort_values(by = 'job_name',ascending = False).head(10)['profession_dir'],
        autopct='%.2f%%',
        startangle = -80,
        pctdistance= 0.8,
        radius=0.8
       )
plt.legend(title = '行业前10招聘数量占比分布',loc = 'lower right',frameon = False)
plt.title('行业招聘分布')
plt.show()

#每个职位的招聘人数,查找开头为数据的职位
_data = list(set([i for i in df['job_name'] if str(i).startswith('数据')]))
df_data = df.query("job_name ==@_data")

#每个去招聘数据量分布
str_df = pd.DataFrame(df_data['zone'].value_counts()).reset_index().sort_values(by = 'zone',ascending = False)
str_df.columns = ['zone','counts']
fig = plt.figure(num = 2,figsize = (10,8))
patches,text,autotext = plt.pie(x = str_df['counts'],labels = str_df['zone'],autopct='%.2f%%',
       shadow = True,startangle = -45,explode = [0.05,0,0,0,0])
#控制扇形图
patches[1].set_hatch('|')
patches[0].set_color('tan')
#控制文本
text[0].set_color('navy')
text[0].set_fontsize('18')
#控制百分比
autotext[0].set_color('forestgreen')
autotext[0].set_fontsize('20')
plt.title('数据分析开头的职位占比')
plt.legend(loc = 'lower right',frameon = False)
plt.show()

import wordcloud
font = r'C:\Windows\Fonts\simhei.ttf'
# font = r'E:\Anaconda3\Lib\site-packages\wordcloud\msyh.ttc'
w = wordcloud.WordCloud(font_path = font,width=400,height=400,background_color='white',
                       stopwords={'海口'})
w.generate(txt1)
plt.imshow(w)
plt.axis('off')
plt.show()
# w.to_file('data2.jpg')