HDFS 中读取数据的方法
方法之一： 通过URL
?
?
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
?
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.io.IOUtils;
?
public class Test {
static {
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
}
/**
* @param args
*/
public static void main(String[] args) {
InputStream in = null;
try {
in = new URL("hdfs://192.168.1.150:9000/tmp/output/part-r-00000").openStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
String line = null;
while( (line = reader.readLine()) != null){
System.out.println(line);
}
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
IOUtils.closeStream(in);
}
}
}
?
方法二： 通过FileSytem 方式
?
?
?
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
?
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
?
public class FileSytemCat {
?
/**
* @param args
* @throws IOException?
*/
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
Configuration conf = new Configuration();
String uri = "hdfs://192.168.1.150:9000/tmp/output/part-r-00000";
FileSystem fs = FileSystem.get(URI.create(uri), conf);
InputStream in = null;
in = fs.open(new Path(uri));
IOUtils.copyBytes(in, System.out,4096, false);
IOUtils.closeStream(in);
}
}
