Hadoop Core 学习笔记(二) lzo文件的写入和读取
本博客属原创文章,转载请注明出处:http://guoyunsky.iteye.com/blog/1266226
? 欢迎加入Hadoop超级群:?180941958? ?
?
? ? ? 压缩是绕不开的话题,因为当今很多程序的压力还是在IO.特别是Hadoop这种分布式存储和运算框架,单台机器的IO,网络通信IO都是压力和挑战.关于Hadoop为什么要用Lzo来压缩而没有选用传统的压缩方法,我这里不再阐述.相关资料很多.有兴趣的可以查看cloudera这篇博客:http://www.cloudera.com/blog/2009/11/hadoop-at-twitter-part-1-splittable-lzo-compression/
? ? ? 这里只是用于读写lzo文件,具体请看代码吧.
?
package com.guoyun.hadoop.io.study;import java.io.BufferedReader;import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.OutputStream;import java.util.ArrayList;import java.util.List;import org.apache.hadoop.conf.Configuration;import com.hadoop.compression.lzo.LzopCodec;/** * 读写Lzo文件 */public class LzoFileStudy { private static Configuration getDefaultConf(){ Configuration conf=new Configuration(); conf.set("mapred.job.tracker", "local"); conf.set("fs.default.name", "file:///"); conf.set("io.compression.codecs", "com.hadoop.compression.lzo.LzoCodec"); return conf; } /** * 写入数据到lzo文件 * * @param destLzoFilePath * @param conf * @param datas */ public static void write2LzoFile(String destLzoFilePath,Configuration conf,byte[] datas){ LzopCodec lzo=null; OutputStream out=null; try { lzo=new LzopCodec(); lzo.setConf(conf); out=lzo.createOutputStream(new FileOutputStream(destLzoFilePath)); out.write(datas); } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ try { if(out!=null){ out.close(); } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } /** * 从lzo文件中读取数据 * * @param lzoFilePath * @param conf * @return */ public static List<String> readLzoFile(String lzoFilePath,Configuration conf){ LzopCodec lzo=null; InputStream is=null; InputStreamReader isr=null; BufferedReader reader=null; List<String> result=null; String line=null; try { lzo=new LzopCodec(); lzo.setConf(conf); is=lzo.createInputStream(new FileInputStream(lzoFilePath)); isr=new InputStreamReader(is); reader=new BufferedReader(isr); result=new ArrayList<String>(); while((line=reader.readLine())!=null){ result.add(line); } } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ try { if(reader!=null){ reader.close(); } if(isr!=null){ isr.close(); } if(is!=null){ is.close(); } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } return result; } /** * @param args */ public static void main(String[] args) { // 生成数据 String dataSource="abcdefghijklmnopqrstuvwxyz0123456789～！＠＃￥％……＆＊（）——＋\r"; dataSource=dataSource.concat(dataSource); dataSource=dataSource.concat(dataSource); dataSource=dataSource.concat(dataSource); String lzoFilePath="./data/test.lzo"; // 写入到lzo文件 write2LzoFile(lzoFilePath,getDefaultConf(),dataSource.getBytes()); StringBuilder sb=new StringBuilder(); // 读取lzo文件 List<String> lines=readLzoFile(lzoFilePath,getDefaultConf()); for(String line:lines){ sb.append(line); sb.append("\r"); } // 数据是否一致 if(sb.toString().equals(dataSource)){ System.out.println(sb.toString()); }else{ System.err.println("Error line:"+sb.toString()); } }}