使用Java API上传数据到HDFS

@Test
public void test1() throws Exception {//构造一个输入流，代表要上传的数据InputStream input = new FileInputStream("d:\\temp\\hadoop-2.7.3.tar.gz");//NameNodeProxies//NameNode//指定NameNode 地址Configuration conf = new Configuration();conf.set("fs.defaultFS", "hdfs://192.168.157.111:9000");//创建HDFS的客户端 DistributedFileSystemFileSystem client = FileSystem.get(conf);//构造一个输出流，指向HDFSOutputStream output = client.create(new Path("/folder1/a.tag.gz"));//缓冲区byte[] buffer = new byte[1024];//长度int len = 0;while( (len = input.read(buffer)) > 0 ) {//写到输出流output.write(buffer, 0, len);}output.flush();output.close();input.close();
}

使用工具类简化后的代码:

@Test
public void test2() throws Exception {//构造一个输入流，代表要上传的数据InputStream input = new FileInputStream("d:\\temp\\hadoop-2.7.3.tar.gz");//指定NameNode 地址Configuration conf = new Configuration();conf.set("fs.defaultFS", "hdfs://192.168.157.111:9000");//创建HDFS的客户端FileSystem client = FileSystem.get(conf);//构造一个输出流，指向HDFSOutputStream output = client.create(new Path("/folder1/b.tag.gz"));//使用工具类简化IOUtils.copyBytes(input, output, 1024);
}

HDFS数据的下载

@Test
public void test1() throws Exception{//构造一个输出流，指向本地OutputStream output = new FileOutputStream("d:\\temp\\x.tar.gz");//指定NameNode的地址Configuration conf = new Configuration();conf.set("fs.defaultFS", "hdfs://192.168.157.111:9000");//创建HDFS的客户端FileSystem client = FileSystem.get(conf);//得到一个输入流InputStream input = client.open(new Path("/folder1/b.tag.gz"));//构造一个缓冲区byte[] buffer = new byte[1024];int len = 0;while((len=input.read(buffer)) > 0) {output.write(buffer, 0, len);}output.flush();output.close();input.close();
}

使用工具类简化后的代码:

@Test
public void test1() throws Exception{//构造一个输出流，指向本地OutputStream output = new FileOutputStream("d:\\temp\\x.tar.gz");//指定NameNode的地址Configuration conf = new Configuration();conf.set("fs.defaultFS", "hdfs://192.168.157.111:9000");//创建HDFS的客户端FileSystem client = FileSystem.get(conf);//得到一个输入流InputStream input = client.open(new Path("/folder1/b.tag.gz"));//构造一个缓冲区byte[] buffer = new byte[1024];int len = 0;while((len=input.read(buffer)) > 0) {output.write(buffer, 0, len);}output.flush();output.close();input.close();
}