文章目录

• 一、 hadoop window配置
• • 1.1 hadoop_home环境变量配置
  • 1.2 Hadoop里的Java路径配置
• 二、IDE远程管理HDFS
• • 1.创建maven，导入pom.xml依赖
  • 2.案例测试

提示：以下是本篇文章正文内容，下面案例可供参考

一、 hadoop window配置

hadoop tar包解压

1.1 hadoop_home环境变量配置

1.2 Hadoop里的Java路径配置

D:\hadoop-3.3.4\etc\hadoop\hadoop-env.cmd文件修改
(因program file 有空格。用PROGRA~1代替)
set JAVA_HOME=C:\PROGRA~1\Java\jdk1.8.0_241

下载windows下连接hadoop需要的文件hadoop.dll和winutils.exe(https://github.com/
下载apache-hadoop-3.1.1-winutils-master)

二、IDE远程管理HDFS

1.创建maven，导入pom.xml依赖

  org.apache.hadoophadoop-client3.3.1

2.案例测试

读取数据（示例）：

package hdfs.demo;import java.io.InputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
public class FileSystemCat {public static void main(String[] args) throws Exception {//创建配置文件对象，加载配置文件中的配置信息//默认读取core-default.xml hdfs-default.xml mapred-defalt.xml yarn-default.xml// 属性优先级：代码中设置>-site.xml>-default.xmlConfiguration conf = new Configuration();// 设置HDFS访问地址conf.set("fs.default.name", "hdfs://192.168.10.101:9820");// 取得FileSystem文件系统实例FileSystem fs = FileSystem.get(conf);//System.out.println(fs.getClass().getName());// 打开文件输入流InputStream in = fs.open(new Path("hdfs:/input/README.txt"));// 输出文件内容IOUtils.copyBytes(in, System.out, 4096, false);// 关闭输入流IOUtils.closeStream(in);}
}

创建目录

package hdfs.demo;import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class CreateDir {public static void main(String[] args) throws IOException {Configuration conf = new Configuration();conf.set("fs.default.name", "hdfs://192.168.10.101:9820");FileSystem hdfs = FileSystem.get(conf);// 创建目录boolean isok = hdfs.mkdirs(new Path("hdfs:/mydir"));if (isok) {System.out.println("创建目录成功!");} else {System.out.println("创建目录失败！");}hdfs.close();}
}

…

package hdfs.demo;import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
/*** HDFS Java API文件操作*/
public class HDFSDemo {public static void main(String[] args) throws Exception {createFile();outFile();deleteFile();copyFromLocalFile();copyToLocalFile();updateFlileProgress();deleteFile();}/*** 定义创建文件方法*/public static void createFile() throws Exception {Configuration conf = new Configuration();conf.set("fs.default.name", "hdfs://192.168.170.133:9000");FileSystem fs = FileSystem.get(conf);// 打开一个输出流FSDataOutputStream outputStream = fs.create(new Path("hdfs:/newfile2.txt"));// 写入文件内容outputStream.write("我是文件内容".getBytes());outputStream.close();fs.close();System.out.println("文件创建成功！");}// 删除文件public static void deleteFile() throws Exception {Configuration conf = new Configuration();conf.set("fs.default.name", "hdfs://192.168.170.133:9000");FileSystem fs = FileSystem.get(conf);Path path = new Path("hdfs:/newfile2.txt");boolean isok = fs.deleteOnExit(path);if (isok) {System.out.println("删除成功!");} else {System.out.println("删除失败！");}fs.close();}// 复制上传本地文件public static void copyFromLocalFile() throws Exception {// 1.创建配置器Configuration conf = new Configuration();conf.set("fs.default.name", "hdfs://192.168.170.133:9000");// 2.取得FileSystem文件系统 实例FileSystem fs = FileSystem.get(conf);// 3.创建可供hadoop使用的文件系统路径Path src = new Path("D:/copy_test.txt"); // 本地目录/文件Path dst = new Path("hdfs:/"); // 目标目录/文件// 4.拷贝上传本地文件（本地文件，目标路径） 至HDFS文件系统中fs.copyFromLocalFile(src, dst);System.out.println("文件上传成功!");}// 监控文件上传进度public static void updateFlileProgress() throws Exception {Configuration conf = new Configuration();conf.set("fs.default.name", "hdfs://192.168.170.133:9000");InputStream in=new BufferedInputStream(new FileInputStream("D:/soft/test.zip"));FileSystem fs = FileSystem.get(conf);//上传文件并监控上传进度FSDataOutputStream outputStream = fs.create(new Path("hdfs:/test.zip"), new Progressable() {public void progress() {//回调方法显示进度System.out.print(".");}});IOUtils.copyBytes(in, outputStream, 4096, false);}// 复制下载文件public static void copyToLocalFile() throws Exception {// 1.创建配置器Configuration conf = new Configuration();conf.set("fs.default.name", "hdfs://192.168.170.133:9000");// 2.取得FileSystem文件系统 实例FileSystem fs = FileSystem.get(conf);// 3.创建可供hadoop使用的文件系统路径Path src = new Path("hdfs:/newfile2.txt");// 目标目录/文件Path dst = new Path("D:/new.txt"); // 本地目录/文件// 4.从HDFS文件系统中拷贝下载文件（目标路径，本地文件）至本地// fs.copyToLocalFile(src, dst);fs.copyToLocalFile(false, src, dst, true);System.out.println("文件下载成功!");}// 查看文件内容并输出public static void outFile() throws Exception {// 1.创建配置器Configuration conf = new Configuration();conf.set("fs.default.name", "hdfs://192.168.170.133:9000");// 2.取得FileSystem文件系统 实例FileSystem fs = FileSystem.get(conf);InputStream in = fs.open(new Path("hdfs:/newfile2.txt"));IOUtils.copyBytes(in, System.out, 4096, false);IOUtils.closeStream(in);}}