HttpClient抓取网页实例
import java.io.BufferedReader;import java.io.BufferedWriter;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStreamReader;import java.io.OutputStreamWriter;import org.apache.commons.httpclient.HttpClient;import org.apache.commons.httpclient.HttpException;import org.apache.commons.httpclient.HttpStatus;import org.apache.commons.httpclient.NameValuePair;import org.apache.commons.httpclient.methods.PostMethod;public class Client {public static void test(String url,String encode,String path){HttpClient httpClient = new HttpClient();//httpClient.getHostConfiguration().setProxy(proxyHost, proxyPort) 可设置代理// post方法PostMethod postMethod = new PostMethod(url);String from = "广州";String to = "成都";//需要的参数名称和值NameValuePair[] paras = new NameValuePair[3];paras[0] = new NameValuePair("from", from);paras[1] = new NameValuePair("to",to);paras[2] = new NameValuePair("act", "3");postMethod.addParameters(paras);postMethod.setRequestHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)");postMethod.setRequestHeader("Content-Type", "application/x-www-form-urlencoded;charset="+encode);postMethod.setRequestHeader("Referer", "http://www.ttkdex.com/");BufferedReader reader = null;BufferedWriter writer = null;try {//响应状态码 成功为200int code = httpClient.executeMethod(postMethod);if(HttpStatus.SC_OK != code){System.out.println("响应不成功");return;}System.out.println("响应成功：code="+code); reader = new BufferedReader(new InputStreamReader(postMethod.getResponseBodyAsStream(), encode)); writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(path), encode)); String line = null; while ((line = reader.readLine()) != null) { writer.write(line+"\r\n"); } reader.close();writer.close();System.out.println("success ! result in file : "+path);} catch (HttpException e){e.printStackTrace();}catch (IOException e) {e.printStackTrace();}finally{//完成，释放连接，释放资源postMethod.releaseConnection();try {if(reader != null){reader.close();}if(writer != null){reader.close();}} catch (IOException e) {e.printStackTrace();}}System.out.println("run over ...");}public static void main(String[] args) {String url = "http://qq.ip138.com/train/train_search.asp";String encode = "gb2312";String savePath = "E:/test.html";test(url,encode,savePath);}}