爬取页面需要登陆才可爬取，这种怎么解决
如题，就是如果要爬取某个页面，但它必须要你在它的登陆页面，登陆后，内容才可以显示出来，请问这种的是怎么解决，谢了。
[解决办法]
我的做法是用WebBrowser模拟用户登录，如果带验证码的，可以用验证码识别算法，或者弹出一个对话框输入验证码，然后再跳转到要爬取的页面
[解决办法]
这个也就是模拟登录，没有验证码的好办.
用抓包工具wireshark看下那个网站登录是post到哪的，需要带哪些参数，登录成功后再请求需要的数据页,最近也有做过类似的功能，代码你可以参考下.
public HttpWebResponse PostData(string strURL, string strArgs, string strReferer, string code, string method, CookieContainer cookieContainer)
{
return PostData(strURL, strArgs, strReferer, code, method, string.Empty,cookieContainer);
}
public HttpWebResponse PostData(string strURL, string strArgs, string strReferer, string code, string method, string contentType, CookieContainer cookieContainer)
{
try
{
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(strURL);
myHttpWebRequest.AllowAutoRedirect = true;
myHttpWebRequest.KeepAlive = true;
myHttpWebRequest.Accept = "application/json, text/javascript, */*";
myHttpWebRequest.Referer = strReferer;
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.96 Safari/537.4";
if (string.IsNullOrEmpty(contentType))
{
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
}
else
{
myHttpWebRequest.ContentType = "contentType";
}
myHttpWebRequest.Method = method;
myHttpWebRequest.Headers.Add("Accept-Encoding", "gzip, deflate,sdch");
if (cookieContainer == null)
{
cookieContainer = new CookieContainer();
}
myHttpWebRequest.CookieContainer = cookieContainer;
byte[] postData = Encoding.GetEncoding(code).GetBytes(strArgs);
myHttpWebRequest.ContentLength = postData.Length;
System.IO.Stream PostStream = myHttpWebRequest.GetRequestStream();
PostStream.Write(postData, 0, postData.Length);
PostStream.Close();
HttpWebResponse response = null;
response = (HttpWebResponse)myHttpWebRequest.GetResponse();
return response;
}
catch (Exception ex)
{
string s = "出错了：" + ex.Message;
return null;
}
}
private void login_ajax()
{
string username = Request.Form["username"];
//登录地址
string LOGIN_URL = "http://mp.weixin.qq.com/cgi-bin/login?lang=zh_CN";
//登录请求来源地址
string LOGIN_REFERER = "http://mp.weixin.qq.com/cgi-bin/loginpage?t=wxm-login&lang=zh_CN";
CookieContainer cookie = new CookieContainer();
if (Session["login_verify_code"] != null)
{
cookie = Session["login_verify_code"] as CookieContainer;
}
string is_update = Request.QueryString["is_update"] ?? "";
string pwd1 = Request.Form["pwd1"];
string pwd2 = Request.Form["pwd2"];
string imgcode = Request.Form["imgcode"];
string register = Request.Form["register"];
string f = Request.Form["f"];
//拼接请求参数
string strArgs = "&username=" + username;
strArgs += "&pwd1=" + pwd1;
strArgs += "&pwd2=" + pwd2;
strArgs += "&imgcode=" + imgcode;
strArgs += "&f="+f;
HttpWebResponse http_response = pt.PostData(LOGIN_URL, strArgs, LOGIN_REFERER, CODE, MOTHED, cookie);
StreamReader reader = new StreamReader(http_response.GetResponseStream(), Encoding.Default);
content = reader.ReadToEnd();
Response.Write(content);
reader.Close();
http_response.Close();
Session["username"]=username;
Session["pwd"] = pwd1;
Session["login_wx"] = cookie;
//此处是登录成功后要取的网页
string temp_url = "http://mp.weixin.qq.com/cgi-bin/userinfopage?t=wxm-setting&lang=zh_CN";
HttpWebResponse http_response_get = pt.GetResponseByGet(temp_url, cookie);
StreamReader reader1 = new StreamReader(http_response_get.GetResponseStream(), Encoding.Default);
string content1 = reader1.ReadToEnd();
}
