前一段时间,公司以前的一个数据采集任务突然之间采集下来的数据都是0了,也就是未登录状态能够获取到的数据,于是猜想肯定是网站的服务升级了,升级了数据接口的逻辑,于是便开始解决此问题。
此采集程序是由.net core开发,采用Quartz定时任务定时采集数据。
下面是解决方法:
1.首先从登录url拿到cookie,然后保存至内存中
2.打开需要爬取的网页时,将拿到的cookie放到打开网页的request中
3.在第2步中ContentType 和UserAgent 需要与登录获取cookie一致
/// <summary>/// 获取Cookie/// </summary>/// <param name="url">登录url,例如http://www.website.com/user/doLogin</param>/// <param name="params">登录参数,例如username=username&pwd=pwd&forever=1</param>/// <returns></returns>public static CookieCollection GetCookieCollection(string url, string params){CookieContainer cc = new CookieContainer();ASCIIEncoding encoding = new ASCIIEncoding();byte[] data = encoding.GetBytes(params);HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);request.Method = "POST";request.ContentType = "application/x-www-form-urlencoded";request.ContentLength = data.Length;request.UserAgent = "Chrome/87.0.4280.66"; Stream newStream = request.GetRequestStream();newStream.Write(data, 0, data.Length);newStream.Close();request.CookieContainer = cc;HttpWebResponse response = (HttpWebResponse)request.GetResponse();CookieCollection cookieCollection = response.Cookies;DateTime dt = DateTime.Now.AddMinutes(365 * 24 * 60);//为cookie添加过期时间for (int i = 0; i < cookieCollection.Count; i++){cookieCollection[i].Expires = dt;}return cookieCollection;}/// <summary>/// 获取网页代码/// </summary>/// <param name="url">需要爬取的数据所在网页url</param>/// <param name="cookieCollection">GetCookieCollection方法获取到的cookie</param>/// <returns></returns>public static string GetWebContent(string url, CookieCollection cookieCollection) {CookieContainer cc = new CookieContainer();cc.Add(cookieCollection);HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);request.CookieContainer = cc;request.ContentType = "application/x-www-form-urlencoded";//与登录一致request.UserAgent = "Chrome/87.0.4280.66"; //与登录一致HttpWebResponse response = (HttpWebResponse)request.GetResponse();Stream stream = response.GetResponseStream();string webContent = new StreamReader(stream, System.Text.Encoding.Default).ReadToEnd();return webContent;}