代码重构~
package com.eetrust.cpm.IPparse;
public class Directory2 {
//private static File[] strPath2;
public static String convertingStr(String lineStr) {
String retVal = "";
return retVal;
}
public static void recursion(String root, Vector<String> vecFile) {
File file = new File(root);
File[] subFile = file.listFiles();
if (subFile.length == 0) {
System.out.println(file.getAbsolutePath() + " is null");
}else{
for (int i = 0; i < subFile.length; i++) {
if (subFile[i].isDirectory()) {
recursion(subFile[i].getAbsolutePath(), vecFile);
} else {
String filename = subFile[i].getName();
vecFile.add(subFile[i].getAbsolutePath());
}
}
}
}
public String GetNowDate(){
String temp_str="";
Date dt = new Date();
//最后的aa表示“上午”或“下午” HH表示24小时制 如果换成hh表示12小时制
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss aa");
temp_str=sdf.format(dt);
return temp_str;
}
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
Map <String,Integer> map = new HashMap<String,Integer>();
IPtest ipTest = new IPtest();
String path = ipTest.getClass().getProtectionDomain().getCodeSource().getLocation().getPath();
//System.out.println(path);
//if (path.indexOf("WEB-INF") > 0) {
//path = path.substring(1, path.indexOf("/WEB-INF/"));
//}
path+="/qqwry";
// 指定纯真数据库的文件名，所在文件夹
IPSeeker ip = new IPSeeker("QQWry.Dat", path);
// 测试IP 58.20.43.13
System.out.println(path);
Vector<String> vecFile = new Vector<String>();
recursion("/opt/haproxy_temp_log", vecFile);
for (String fileName : vecFile) {
System.out.println(fileName);
Pattern pattern = Pattern.compile("\\S*\\d+\\.\\d+\\.\\d+\\.\\d+$");
Matcher m2 = pattern.matcher(fileName);
if(m2.matches()!=false){
System.out.println("不需要处理!@");
}else if(fileName.indexOf("cache.rightgo.net")!=-1){
System.out.println("不需要分析！");
}else {
BufferedReader br = new BufferedReader(new FileReader(fileName));
//System.out.println(i);
String line = null;
while ((line = br.readLine()) != null){
Pattern p = Pattern.compile(":\\s([^:]*):");//这个匹配还是有点取巧了。
//可以这样":\\s*(\\d+\\.\\d+\\.\\d+\\.\\d+)\\s*:"安全一点
Matcher m = null;
if(!"".equals(line)){
m = p.matcher(line);
while(m.find()){
String key1 = ip.getIPLocation(m.group(1).trim()).getCountry();
//北京市、上海市、天津市、重庆市。
// System.out.println(key);
if(key1.indexOf("黑龙江")!=-1){
key1=key1.substring(0, 3);
}else if(key1.indexOf("省")!=-1^key1.indexOf("市")!=-1){
key1=key1.substring(0, 2);
}else if(key1.indexOf("香港")!=-1){
key1=key1.substring(0, 2);
}else if(key1.indexOf("澳门")!=-1){
key1=key1.substring(0, 2);
}
else{
key1="其它";
}
String key2 = ip.getIPLocation(m.group(1).trim()).getArea();
if(key2.indexOf("电信")!=-1){
key2="电信";
}else if(key2.indexOf("移动")!=-1){
key2="移动";
}else if(key2.indexOf("网通")!=-1^key2.indexOf("联通")!=-1){
key2="联通";
}else if(key2.indexOf("学")!=-1^key2.indexOf("教")!=-1){
key2="教育网";
}else{
key2="其它";
}
String fileName2;
if(fileName.indexOf(":80")!=-1){
fileName2 = fileName.substring(fileName.lastIndexOf(File.separator)+1,fileName.lastIndexOf(":80"));
}else{
fileName2 = fileName.substring(fileName.lastIndexOf(File.separator)+1,fileName.length());
}
//System.out.println(fileName2);
String key = key1+key2+fileName2;
//String key = key1+key2+fileName;
// System.out.println(key);
// System.out.println(files[i].getName());
if (map.containsKey(key)) {
Integer temp2 = map.get(key);
map.put(key, ++temp2);
} else {
map.put(key, new Integer(1));
}
}// end of while;
}// end of if;
}//end of while;
br.close();
}
}
try {
Class.forName("com.mysql.jdbc.Driver");
//定义所要用到的三个数据库应用对象
// Connection con=null; //连接对象
// Statement sql=null; //Statement对象（SQL语句）
// ResultSet rs=null; //结果集对象
//进行数据源的连接
Connection con;
try{
con = DriverManager.getConnection ("jdbc:mysql://localhost/cdn_web_user_stats?user=root&password=rjkj@rjkj&useUnicode=true&characterEncoding=utf8");
List<Map.Entry<String, Integer>> list2 = new LinkedList<Map.Entry<String, Integer>>();
list2.addAll(map.entrySet());
for(Iterator<Map.Entry<String, Integer>> ite = list2.iterator(); ite.hasNext();) {
Map.Entry<String, Integer> map2 = ite.next();
System.out.println(map2.getKey());
String n;
if(map2.getKey().indexOf("黑")!=-1 ){
System.out.println(map2.getKey());
n=map2.getKey().substring(5,map2.getKey().length());
if(map2.getKey().indexOf("教")!=-1){
n=map2.getKey().substring(6,map2.getKey().length());
}
}else if(map2.getKey().indexOf("教")!=-1){
n=map2.getKey().substring(5,map2.getKey().length());
}
else{
n = map2.getKey().substring(4,map2.getKey().length());
}
System.out.println(n);
Statement stmt = con.createStatement();
String sql2 = "CREATE TABLE IF NOT EXISTS `cdn_web_user_stats`.`"+n+"` ("
+" `id` int(11) NOT NULL AUTO_INCREMENT,"
+" `date` date NOT NULL,"
+"`zone` char(50) NOT NULL,"
+"`nettype` varchar(50) NOT NULL,"
+"`cnt` int(32) unsigned NOT NULL,"
+"PRIMARY KEY (`id`),"
+" UNIQUE KEY datezone (`date`,`zone`,`nettype`),"
+"KEY `cnt` (`cnt`)"
+" ) ENGINE=MyISAM DEFAULT CHARSET=utf8";
stmt.execute(sql2);
String zone1;
if(map2.getKey().indexOf("黑")!=-1){
zone1 = map2.getKey().substring(0, 3);
}else{
zone1 = map2.getKey().substring(0, 2);
}
String net;
if(map2.getKey().indexOf("黑")!=-1){
net = map2.getKey().substring(3, 5);
if(map2.getKey().indexOf("教")!=-1){
net = map2.getKey().substring(3, 6);
}
} else if(map2.getKey().indexOf("学")!=-1 ^ map2.getKey().indexOf("教")!=-1){
net = map2.getKey().substring(2, 5);
} else{
net = map2.getKey().substring(2, 4);
}
int value = map2.getValue();
String sql="insert into `cdn_web_user_stats`.`"+n+"`(date,zone,nettype,cnt) values(?,?,?,?)";
System.out.println(sql);
PreparedStatement pstmt = con.prepareStatement(sql);
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DATE, -1); //得到前一天
Date date = calendar.getTime();
DateFormat df = new SimpleDateFormat("yyyy-MM-dd");
String n1=df.format(date);
pstmt.setString(1, n1);
pstmt.setString(2,zone1);
pstmt.setString(3, net);
pstmt.setInt(4, value);
if(pstmt.executeUpdate()>0) //判断结果集是否为空，如果不为空则表示有记录
{
System.out.println("添加成功！");
}
else {
System.out.println("添加失败！");
}
}//end of for
// }//end of foe
}catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
// end of main
[解决办法]
很明显内存溢出。
代码没有细看。不过你的处理方法是把所有的文件的数据读入，放在Map容器中。然后再存到ＤＢ。
当文件过大时，Map容器放不下，肯定会内存溢出。
我在上面已经大致告诉你思路。我再细化一下．
１。从文件中读１０００件存入Ｍａｐ
２．把Map存入ＤＢ
３．Clean Map
４．继续读下一个１０００件．
