本文共 4694 字,大约阅读时间需要 15 分钟。
前提已经安装配置好hadoop2.2,安装请参考我另一篇文章
排版太辛苦,先弄个简易版出来,只要看显示的那两个类就够了,其他部分是我项目无关HDFS文件读写(请见谅)
pom.xml配置如下
4.0.0 storage platform.edu.storage 0.0.1-SNAPSHOT explorer-impl ${project.artifactId} jar Hadoop HDFS explorer实现 org.apache.hadoop hadoop-client 2.2.0
HDFSServer.java代码如下
-----------------------------------------------------------
package platform.edu.explorer.hdfs.server; import java.io.IOException;import java.net.URI; import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.mapred.JobConf; /** * HDFS服务器 * * @author hjn * @version 1.0 2013-11-20 */public class HDFSServer {private static Configuration configuration;private static FileSystem fileSystem;private static final String HDFS_URL = "hdfs://192.168.1.210:8020"; /** * HDFS服务器读取初始化 */private static void init() {try {configuration= new JobConf(HDFSServer.class);fileSystem = FileSystem.get(URI.create(HDFS_URL), configuration);} catch (IOException e) {System.out.println("读取服务器失败");e.printStackTrace();}}public static FileSystem getFileSystem(){if(fileSystem==null){init();}return fileSystem;} } --------------------------------------------------------------------------------------------------------------------------------------------------HDFS.java代码如下--------------------------package platform.edu.explorer.hdfs; import java.io.File;import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.util.ArrayList;import java.util.List; import org.apache.commons.compress.utils.IOUtils;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path; import platform.edu.explorer.HadoopFileSystem;import platform.edu.explorer.hdfs.server.HDFSServer; /** * HDFS(分布式文件存储系统操作类). * * @author hjn * @version 1.0 2013-11-22 */ public class HDFS implements HadoopFileSystem {/** * 文件系统类 */private FileSystem fileSystem; /** * 无参数构造方法 */public HDFS() { init();} /** * 初始化 */private void init() { fileSystem = HDFSServer.getFileSystem();} /** * 获取HDFS指定目录下文件状态列表 * * @param dirPath指定目录路径 * @return fileStatusList * @throws FileNotFoundException * @throws IOException */public FileStatus[] getFileStatus(Path path) throws FileNotFoundException,IOException { FileStatus[] fileStatusList = fileSystem.listStatus(path); return fileStatusList;} /** * 获取指定目录列表路径 * * @param dirPath */public Listdir(String dirPath) throws IOException { List fileList = null; Path path = new Path(dirPath); if (fileSystem.exists(path)) { fileList = new ArrayList (); FileStatus[] list = this.getFileStatus(path); for (FileStatus fileStatus : list) { fileList.add(fileStatus.getPath().toString()); } } else { System.out.println("目录不存在"); } return fileList;} /** * 获取文件 * * @param filePath * @return * @throws IOException */public InputStream getFile(String filePath) throws IOException {Path path = new Path(filePath);return fileSystem.open(path);} /** * 更改HDSF文件名称 * * @param fileOldName * @param fileNewName * @return boolean:是否更名字成功 * @throws IOException */public boolean rename(String src, String dst) throws IOException { Path srcPath = new Path(src); if (fileSystem.exists(srcPath)) { Path dstPath = new Path(dst); return fileSystem.rename(srcPath, dstPath); } System.out.println("原文件不存在"); return false;} /** * 创建HDFS目录 * * @param dir */public boolean createDir(String dir) throws IOException { Path path = new Path(dir); if (fileSystem.exists(path)) { System.out.println("此目录已经存在不需要再创建"); return true; } return fileSystem.mkdirs(path);} /** * 上传本地文件到HDFS(注意是服务器本地硬盘,非客户端硬盘)) * * @return * * @throws IOException */@Overridepublic void uploadLocalFile(String localFileSrc, String HDFSFileDst) throws IOException { Path src = new Path(localFileSrc); Path dst = new Path(HDFSFileDst); fileSystem.copyFromLocalFile(src, dst);}/** * 批量上传本地文件到HDFS * @param localFileSrcs本地文件列表 * @param HDFSFileDst * @throws IOException */ public void uploadLocalFile(String[] localFileSrcs,String HDFSFileDst) throws IOException{ Path dstPath=new Path(HDFSFileDst); Path[] paths=new Path[localFileSrcs.length]; for (int i=0;i