用C#+Selenium+ChromeDriver 爬取网页，完美模拟真实的用户浏览行为

背景

Selenium是一个用于Web应用程序测试的工具。Selenium测试直接运行在浏览器中，就像真正的用户在操作一样。而对于爬虫来说，使用Selenium操控浏览器来爬取网上的数据那么肯定是爬虫中的杀手武器。这里，我将介绍selenium + 谷歌浏览器的一般使用。

需求

在平常的爬虫开发中，有时候网页是一堆js堆起来的代码，涉及很多异步计算，如果是普通的http 控制台请求，那么得到的源文件是一堆js ，需要自己在去组装数据，很费力；但是采用Selenium+ChromeDriver可以达到所见即所得的完美效果。

实现方式

项目结构：为了方便使用，用的winform程序，附nuget包

以下是form1.cs的代码，这里就只放关键方法代码了。需要安装最新的chrome浏览器+代码中使用的chromedriver是 v2.9.248315

    private void crawlingWebFunc(){SetText("\r\n开始尝试...");List<testfold> surls = new List<testfold>();string path = System.Environment.CurrentDirectory + "\\图片url\\";DirectoryInfo root = new DirectoryInfo(path);DirectoryInfo[] dics = root.GetDirectories();foreach (var itemdic in dics){string txt = "";StreamReader sr = new StreamReader(itemdic.FullName + "\\data.txt");while (!sr.EndOfStream){string str = sr.ReadLine();txt += str;// + "\n";}sr.Close();surls.Add(new testfold() { key = itemdic.FullName, picurl = txt });}ChromeDriverService service = ChromeDriverService.CreateDefaultService(System.Environment.CurrentDirectory);//  service.HideCommandPromptWindow = true;ChromeOptions options = new ChromeOptions();options.AddArguments("--test-type", "--ignore-certificate-errors");options.AddArgument("enable-automation");//   options.AddArgument("headless");//  options.AddArguments("--proxy-server=http://user:password@yourProxyServer.com:8080");using (IWebDriver driver = new OpenQA.Selenium.Chrome.ChromeDriver(service, options, TimeSpan.FromSeconds(120))){driver.Url = "https://www.1688.com/";Thread.Sleep(200);try{int a = 1;foreach (var itemsurls in surls){SetText("\r\n第" + a.ToString() + "个");driver.Navigate().GoToUrl(itemsurls.picurl);//登录if (driver.Url.Contains("login.1688.com")){SetText("\r\n需要登录，开始尝试...");trylogin(driver); //尝试登录完成//再试试driver.Navigate().GoToUrl("https://s.1688.com/youyuan/index.htm?tab=imageSearch&imageType=oss&imageAddress=cbuimgsearch/eWXC7XHHPN1607529600000&spm=");if (driver.Url.Contains("login.1688.com")){//没办法退出SetText("\r\n退出，换ip重试...");return;}}//鼠标放上去的内容因为页面自带只能显示一个的原因 没办法做到全部显示 然后在下载 只能是其他方式下载//  var elements = document.getElementsByClassName('hover-container');//  Array.prototype.forEach.call(elements, function(element) {//  element.style.display = "block";//   console.log(element);//  });//   IJavaScriptExecutor js = (IJavaScriptExecutor)driver;//    var sss = js.ExecuteScript(" var elements = document.getElementsByClassName('hover-container');  Array.prototype.forEach.call(elements, function(element) {  console.log(element); element.setAttribute(\"class\", \"测试title\");  element.style.display = \"block\";  console.log(element); });");Thread.Sleep(500);var responseModel = Write(itemsurls.key, driver.PageSource, Pagetypeenum.列表);Thread.Sleep(500);int i = 1;foreach (var offer in responseModel?.data?.offerList ?? new List<OfferItemModel>()){driver.Navigate().GoToUrl(offer.information.detailUrl);string responseDatadetail = driver.PageSource;Write(itemsurls.key, driver.PageSource, Pagetypeenum.详情);SetText("\r\n第" + a.ToString() + "-" + i.ToString() + "个");Thread.Sleep(500);i++;}}}catch (Exception ex){CloseChromeDriver(driver);throw;}}}

  #region 异常  退出chromedriver[DllImport("user32.dll", EntryPoint = "FindWindow")]private extern static IntPtr FindWindow(string lpClassName, string lpWindowName);[DllImport("user32.dll", EntryPoint = "SendMessage")]public static extern int SendMessage(IntPtr hWnd, int Msg, int wParam, int lParam);public const int SW_HIDE = 0;public const int SW_SHOW = 5;[DllImport("user32.dll", EntryPoint = "ShowWindow")]public static extern int ShowWindow(IntPtr hwnd, int nCmdShow);/// <summary>/// 获取窗口句柄/// </summary>/// <returns></returns>public IntPtr GetWindowHandle(){string name = (Environment.CurrentDirectory + "\\chromedriver.exe");IntPtr hwd = FindWindow(null, name);return hwd;}/// <summary>/// 关闭chromedriver窗口/// </summary>public void CloseWindow(){try{IntPtr hwd = GetWindowHandle();SendMessage(hwd, 0x10, 0, 0);}catch { }}/// <summary>/// 退出chromedriver/// </summary>/// <param name="driver"></param>public void CloseChromeDriver(IWebDriver driver){try{driver.Quit();driver.Dispose();}catch { }CloseWindow();}#endregion 异常  退出chromedriver

　效果