hadoop版本为java
hadoop version Hadoop 2.10.1 Subversion https://github.com/apache/hadoop -r 1827467c9a56f133025f28557bfc2c562d78e816 Compiled by centos on 2020-09-14T13:17Z Compiled with protoc 2.5.0 From source with checksum 3114edef868f1f3824e7d0f68be03650
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.10.1</version>
</dependency>
package com.definesys.hadoop;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import java.io.FileInputStream;
import java.io.IOException;
/**
* @Description:
* @author: jianfeng.zheng
* @since: 2020/12/14 12:36 上午
* @history: 1.2020/12/14 created by jianfeng.zheng
*/
public class HDFS {
public static void main(String[] cmd) throws IOException {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://master:9000/");
// conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName());
System.setProperty("HADOOP_USER_NAME", "hadoop");
FileSystem fs = FileSystem.get(conf);
Path dst = new Path("hdfs://master:9000/demo/hello.txt");
FSDataOutputStream os = fs.create(dst);
FileInputStream is = new FileInputStream("/root/hello.txt");
IOUtils.copy(is, os);
is.close();
os.close();
fs.close();
}
}
若是是web应用，通常会打包为war或者ear，无论是哪一种，这两种包格式都会把依赖包打进去，所以不用作特殊处理，若是须要本地运行，那么须要借助两个插件，把如下配置信息复制到pom.xml中node
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.6</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>com.definesys.hadoop.HDFS</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
maven-jar-plugin会根据配置生成MANIFEST.MF文件，MANIFEST.MF文件记录运行类信息，依赖信息，相似如下这样git
Manifest-Version: 1.0 Archiver-Version: Plexus Archiver Built-By: asan Class-Path: lib/hadoop-client-2.10.1.jar .... Created-By: Apache Maven 3.6.3 Build-Jdk: 1.8.0_161 Main-Class: com.definesys.hadoop.HDFS
classpathPrefix指定了依赖jar包所在的路径为lib，maven-dependency-plugin插件负责将依赖包所有copy到指定路径下，这里指定了${project.build.directory}/lib目录，和classpathPrefix对应，打包完成后执行如下命令便可github
java -jar hadoop-hdfs-1.0.jar #或者手动指定运行类 java -cp hadoop-hdfs-1.0.jar com.definesys.hadoop.HDFS
打包还有一个插件maven-assembly-plugin，不建议使用这个插件进行打包，缘由是这个插件会将全部依赖解压放到一个jar包里，hadoop有些机制是经过spi实现，解压后会形成配置文件覆盖的状况
package com.definesys.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.FileInputStream;
import java.io.IOException;
/**
* @Description:
* @author: jianfeng.zheng
* @since: 2020/12/14 12:36 上午
* @history: 1.2020/12/14 created by jianfeng.zheng
*/
public class HDFS {
public static void main(String[] cmd) throws IOException {
HDFS hdfs = new HDFS();
hdfs.mkdir("/hdfsDemo");
hdfs.putFile("/root/hello.txt", "/hdfsDemo");
hdfs.dowloadFile("/hdfsDemo/hello.txt", "/root/hello-hdfs.txt");
hdfs.deleteFile("/hdfsDemo");
}
public boolean mkdir(String path) throws IOException {
FileSystem fs = this.getHDFSFileSystem();
return fs.mkdirs(new Path(path));
}
public void putFile(String localPath, String hdfsPath) throws IOException {
this.getHDFSFileSystem().copyFromLocalFile(new Path(localPath), new Path(hdfsPath));
}
public void deleteFile(String path) throws IOException {
this.getHDFSFileSystem().delete(new Path(path), true);
}
public void dowloadFile(String hdfsPath, String localPath) throws IOException {
this.getHDFSFileSystem().copyToLocalFile(new Path(hdfsPath), new Path(localPath));
}
private FileSystem getHDFSFileSystem() {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://master:9000/");
System.setProperty("HADOOP_USER_NAME", "hadoop");
try {
FileSystem fs = FileSystem.get(conf);
return fs;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
Exception in thread "main" org.apache.hadoop.security.AccessControlException: Permission denied: user=root, access=WRITE, inode="/":hadoop:supergroup:drwxr-xr-x
at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:350)
at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:251)
at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:189)
HDFS文件系统权限和Linux相似，不一样的用户对文件操做权限不同，若是代码中没有指定用户名，那么就用执行程序的操做系统做为用户名，在这里是root，咱们能够看下hdfs的文件权限web
$ hadoop fs -ls / Found 5 items drwxr-xr-x - asan supergroup 0 2020-12-16 10:07 /001 drwx-w---- - hadoop supergroup 0 2020-12-07 10:54 /tmp drwxr-xr-x - hadoop supergroup 0 2020-12-07 11:05 /user # 根路径权限 $ hadoop fs -ls -d / drwxr-xr-x - hadoop supergroup 0 2020-12-18 00:42 /
有几个解决方案apache
$ hadoop fs -chmod 777 /demo $ hadoop fs -ls -d /demo drwxrwxrwx - hadoop supergroup 0 2020-12-18 00:46 /demo
在master节点加入如下配置centos
<property>
<name>dfs.permissions.enabled</name>
<value>false</value>
</property>
System.setProperty("HADOOP_USER_NAME", "hadoop");
代码需在执行hdfs操做以前加入