摘要
本报告利用Java和Selenium爬虫技术获取数据，并使用ECharts库对薪资数据进行可视化分析，旨在探究不同经验和学历的薪资分布情况。
数据来源
数据来源于Boss直聘，使用Java结合Selenium库进行数据抓取。
- 数据总数：约2000家企业数据
- 数据类型：java岗位、全栈、前端
- 数据地区：深圳、广州
数据清洗
- 比如15-30K·13薪，清洗为3个字段分别存储
UPDATE boss_index
SETsalaryLowest = (SUBSTRING_INDEX(salaryDesc, '-', 1)),salaryHighest = (SUBSTRING_INDEX(SUBSTRING_INDEX(salaryDesc, '-', -1), 'K', 1)),salaryMonth = (CASEWHEN salaryDesc LIKE '%·%' THENREPLACE(SUBSTRING_INDEX(salaryDesc, '·', -1), '薪', '')ELSENULLEND);
数据分析
- 不同学历、不同经验、不同地区薪资分布，使用中位数和众数进行可以实话展示
- 中位数
- 众数
结果展示
tips:数据y轴大于100，结果为xx元/天
核心代码
爬虫
ChromeOptions ops = new ChromeOptions();
ops.addArguments("--remote-allow-origins=*");
System.setProperty("webdriver.chrome.driver", "driver/chromedriver.exe"); //chromedriver.exe存放的路径
System.setProperty("webdriver.chrome.whitelistedIps", "");
ChromeDriver driver = new ChromeDriver(ops);
driver.get("https://www.zhipin.com/web/geek/job?query=%E5%85%A8%E6%A0%88%E5%B7%A5%E7%A8%8B%E5%B8%88&city=101280100");
driver.manage().window().maximize();
数据分析sql
中位数
<select id="getModeSalaryHighest" resultType="com.example.springboot.dto.MedianSalaryResultDTO">SELECTtag AS group_tag_inner,salaryHighest AS mode_salaryHighestFROM(SELECTtag,salaryHighest,COUNT(*) AS countFROMboss_indexWHEREsalaryHighest IS NOT NULLGROUP BYtag,salaryHighest) AS salary_highest_countsJOIN(SELECTtag AS tag_max_count,MAX(count) AS max_countFROM(SELECTtag,salaryHighest,COUNT(*) AS countFROMboss_indexWHEREsalaryHighest IS NOT NULL<if test="jobName != null">and jobName like concat('%', #{jobName}, '%')</if><if test="areaDistrict != null">and areaDistrict like concat('%', #{areaDistrict}, '%')</if><if test="educationLabel != null">and education_label like concat('%', #{educationLabel}, '%')</if>GROUP BYtag,salaryHighest) AS subqueryGROUP BYtag) AS max_count_highest ON salary_highest_counts.tag = max_count_highest.tag_max_count AND salary_highest_counts.count = max_count_highest.max_countGROUP BYsalary_highest_counts.tag,salary_highest_counts.salaryHighest</select>
众数
<select id="getMedianSalarieshigh" resultMap="MedianSalaryResultMap">SELECTtag AS group_tag_inner,salaryHighest AS median_salaryHighestFROM(SELECTtag,salaryHighest,@rowindex := IF(@group_tag = tag, @rowindex + 1, 1) AS rowindex, -- 按tag分组累加行号@group_tag := tag AS group_tag -- 更新tagFROMboss_index,(SELECT @rowindex := 0, @group_tag := '') var_init -- 初始化变量WHEREsalaryHighest IS NOT NULLORDER BYtag,salaryHighest) AS ranked_salariesJOIN(SELECTtag AS tag_total_rows,COUNT(*) AS total_rowsFROMboss_indexWHEREsalaryHighest IS NOT NULL<if test="jobName != null">and jobName like concat('%', #{jobName}, '%')</if><if test="areaDistrict != null">and areaDistrict like concat('%', #{areaDistrict}, '%')</if><if test="educationLabel != null">and education_label like concat('%', #{educationLabel}, '%')</if>GROUP BYtag) AS total_rows ON ranked_salaries.tag = total_rows.tag_total_rowsWHERErowindex IN (FLOOR((total_rows + 1) / 2), FLOOR((total_rows + 2) / 2)) -- 使用总行数变量来确定中间的行号GROUP BYgroup_tag_inner</select>
