Java爬网页步骤

Java爬网页方法

/*Function name:    myGetHttpFile2Description:      爬网页用Input:            URL 例如:http://www.126.comOutput:           字符串,网页的HTML*/  public String myGetHttpFile2(String url){    String authentication=null;    ArrayList al=new ArrayList();    String PageURL = url;                PageURL = url ;        ArrayList urlCollection=new ArrayList();        Hashtable ht=new Hashtable();        //代理，如果要穿过代理将下面注释取消        //ht=System.getProperties();        //authentication = "Basic " + new sun.misc.BASE64Encoder().encode("代理的用户:代理的密码".getBytes());         //ht.put("proxySet", "true");         //ht.put("proxyHost", "172.16.20.2");         //ht.put("proxyPort", "80");            String urlSource=PageURL;        StringBuffer htmlBuffer=new StringBuffer();        String returnStr=null;                int codeStart=0;        int codeEnd=0;        int linkCount=0;                String getURL=null;        String realURL=null;        String urlText=null;        int from=0;        int lenURL=0;        int firstLink=0;        int endLink=0;        boolean isNewsLink=false;                try        {                    URL su = new URL (urlSource);            URLConnection conn = su.openConnection();             //conn.setRequestProperty("Proxy-Authorization", authentication);                        InputStream imageSource=new URL(urlSource).openStream();            int ch;            while((ch=imageSource.read())>-1)            {                htmlBuffer.append((char)ch);            }            imageSource.close();            returnStr= new String(htmlBuffer);            returnStr=new String(returnStr.getBytes("ISO8859_1"),"GBK");        }        catch(Exception e)        {        }                if(returnStr!=null){           return returnStr ;        }else{           return "empty" ;        }   }