2011.06.29——— Jsoup HttpClient 抓取网络上的图片
2011.06.29——— Jsoup HttpClient 抓取网络上的图片
参考：http://www.iteye.com/topic/1106648
http://www.ibm.com/developerworks/cn/java/j-lo-jsouphtml/index.html?ca=drs-
jsoup 官方网站：http://jsoup.org
需要的主要jar包
httpclient-4.0.1jar jsoup-1.5.2.jar
主要代码 如下
Exmaple3.java
package com.th.spider.test;import java.io.BufferedOutputStream;import java.io.FileOutputStream;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import org.apache.http.HttpEntity;import org.apache.http.HttpResponse;import org.apache.http.HttpStatus;import org.apache.http.client.HttpClient;import org.apache.http.client.methods.HttpGet;import org.apache.http.impl.client.DefaultHttpClient;import org.apache.http.params.CoreConnectionPNames;import org.apache.http.util.EntityUtils;import org.jsoup.Connection;import org.jsoup.Jsoup;import org.jsoup.nodes.Document;import org.jsoup.nodes.Element;import org.jsoup.select.Elements;public class Exmaple3 {private static final Log log = LogFactory.getLog(Exmaple3.class);/** * 抓取图片存放目录 */private static final String PIC_DIR = "/home/li/pic";/** * 链接超时 */private static final int TIME_OUT = 5000;static void go3(String url) throws Exception { Connection conn= Jsoup.connect(url); Document doc = conn.get(); Elements links = doc.select("div.piclist img[src]"); for(int i=0;i<links.size();i++){ Element element = links.get(i); final String imgUrl = element.attr("src"); log.info(imgUrl); Thread.sleep(500); new Thread(new Runnable() { public void run() { try { save(imgUrl); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } }).start(); } }static void go2(String url) throws Exception { Connection conn= Jsoup.connect(url); Document doc = conn.get(); Elements links = doc.select("div.cc a[href]"); for(int i=0;i<links.size();i++){ Element element = links.get(i); final String dirUrl = "http://www.3lian.com"+element.attr("href"); log.info(dirUrl); Thread.sleep(500); new Thread(new Runnable() { public void run() { try { Connection conn= Jsoup.connect(dirUrl); Document doc = conn.get(); Elements images = doc.select("div.mb_jjnr img[src]"); for(int j=0;j<images.size();j++){ Element img = images.get(j); String imgUrl = img.attr("src"); log.info(imgUrl); save(imgUrl); } } catch (Exception e) { e.printStackTrace(); } } }).start(); } }/** * 处理帖子URL * @param url * @throws Exception */static void go(String url) throws Exception {// JSOP创建链接Connection conn = Jsoup.connect(url);// 请求返回整个文档对象Document doc = conn.post();// 选择所有class=zoom 的img标签对象Elements imgs = doc.select("img[class=zoom]");// 循环每个img标签for (int i = 0; i < imgs.size(); i++) {Element img = imgs.get(i);// 取得图片的下载地址String picURL = doc.baseUri() + img.attr("file");log.info(picURL);// 保存图片save(picURL);}}//<img src="static/image/common/none.gif" file="data/attachment/forum/201105/08/174412nz3jq4z90s33s2t0.jpg" width="770" onclick="zoom(this, this.src)" id="aimg_180565" onmouseover="showMenu({'ctrlid':this.id,'pos':'12'})" alt="2011.06.29——— Jsoup HttpClient 抓取网络下的图片" title="img_src_29620.jpg" />//doc.select("img[class=zoom]")/** * 保存图片 * @param url * @param i * @throws Exception */static void save(String url) throws Exception {String fileName = url.substring(url.lastIndexOf("/"));String filePath = PIC_DIR + "/" + fileName;BufferedOutputStream out = null;byte[] bit = getByte(url);if (bit.length > 0) {try {out = new BufferedOutputStream(new FileOutputStream(filePath));out.write(bit);out.flush();log.info("Create File success! [" + filePath + "]");} finally {if (out != null)out.close();}}}/** * 获取图片字节流 * @param uri * @return * @throws Exception */static byte[] getByte(String uri) throws Exception {HttpClient client = new DefaultHttpClient();client.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, TIME_OUT);HttpGet get = new HttpGet(uri);get.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, TIME_OUT);try {HttpResponse resonse = client.execute(get);if (resonse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {HttpEntity entity = resonse.getEntity();if (entity != null) {return EntityUtils.toByteArray(entity);}}} catch (Exception e) {e.printStackTrace();} finally {client.getConnectionManager().shutdown();}return new byte[0];}public static void main(String[] args) throws Exception {// 开始抓取图片 go2("http://www.3lian.com/gif/more/03/0301.html");//go3("http://www.ivsky.com/tupian/nvxing_gouwu_qingjing_v6969/");}}