java操作hdfs文件、文件夹

工具类HdfsUtils.java，及测试用例代码如下：

HdfsUtils.java

1. package com.xy6.demo.utils;

2.  

3. import java.io.ByteArrayInputStream;

4. import java.io.IOException;

5. import java.io.InputStream;

6. import java.io.OutputStream;

7. import java.net.URI;

8. import java.util.ArrayList;

9. import java.util.List;

10.  

11. import org.apache.commons.lang.StringUtils;

12. import org.apache.hadoop.conf.Configuration;

13. import org.apache.hadoop.fs.FSDataInputStream;

14. import org.apache.hadoop.fs.FSDataOutputStream;

15. import org.apache.hadoop.fs.FileStatus;

16. import org.apache.hadoop.fs.FileSystem;

17. import org.apache.hadoop.fs.Path;

18. import org.apache.zookeeper.common.IOUtils;

19.  

20. /**

21. * operate hdfs file or directory util class

22. *

23. * @author zhang

24. * @since 2016-09-26

25. *

26. */

27. public class HdfsUtils {

28.  

29. public static String uri = "hdfs://192.168.1.118:9000";

30.  

31. /**

32. * make a new dir in the hdfs

33. *

34. * @param dir the dir may like '/tmp/testdir'

35. * @return boolean true-success, false-failed

36. * @exception IOException something wrong happends when operating files

37. */

38. public static boolean mkdir(String dir) throws IOException {

39. if (StringUtils.isBlank(dir)) {

40. return false;

41. }

42. dir = uri + dir;

43. Configuration conf = new Configuration();

44. FileSystem fs = FileSystem.get(URI.create(dir), conf);

45. if (!fs.exists(new Path(dir))) {

46. fs.mkdirs(new Path(dir));

47. }

48.  

49. fs.close();

50. return true;

51. }

52.  

53. /**

54. * delete a dir in the hdfs.

55. * if dir not exists, it will throw FileNotFoundException

56. *

57. * @param dir the dir may like '/tmp/testdir'

58. * @return boolean true-success, false-failed

59. * @exception IOException something wrong happends when operating files

60. *

61. */

62. public static boolean deleteDir(String dir) throws IOException {

63. if (StringUtils.isBlank(dir)) {

64. return false;

65. }

66. dir = uri + dir;

67. Configuration conf = new Configuration();

68. FileSystem fs = FileSystem.get(URI.create(dir), conf);

69. fs.delete(new Path(dir), true);

70. fs.close();

71. return true;

72. }

73.  

74. /**

75. * list files/directories/links names under a directory, not include embed

76. * objects

77. *

78. * @param dir a folder path may like '/tmp/testdir'

79. * @return List<String> list of file names

80. * @throws IOException file io exception

81. */

82. public static List<String> listAll(String dir) throws IOException {

83. if (StringUtils.isBlank(dir)) {

84. return new ArrayList<String>();

85. }

86. dir = uri + dir;

87. Configuration conf = new Configuration();

88. FileSystem fs = FileSystem.get(URI.create(dir), conf);

89. FileStatus[] stats = fs.listStatus(new Path(dir));

90. List<String> names = new ArrayList<String>();

91. for (int i = 0; i < stats.length; ++i) {

92. if (stats[i].isFile()) {

93. // regular file

94. names.add(stats[i].getPath().toString());

95. } else if (stats[i].isDirectory()) {

96. // dir

97. names.add(stats[i].getPath().toString());

98. } else if (stats[i].isSymlink()) {

99. // is s symlink in linux

100. names.add(stats[i].getPath().toString());

101. }

102. }

103.  

104. fs.close();

105. return names;

106. }

107.  

108. /*

109. * upload the local file to the hds,

110. * notice that the path is full like /tmp/test.txt

111. * if local file not exists, it will throw a FileNotFoundException

112. *

113. * @param localFile local file path, may like F:/test.txt or /usr/local/test.txt

114. *

115. * @param hdfsFile hdfs file path, may like /tmp/dir

116. * @return boolean true-success, false-failed

117. *

118. * @throws IOException file io exception

119. */

120. public static boolean uploadLocalFile2HDFS(String localFile, String hdfsFile) throws IOException {

121. if (StringUtils.isBlank(localFile) || StringUtils.isBlank(hdfsFile)) {

122. return false;

123. }

124. hdfsFile = uri + hdfsFile;

125. Configuration config = new Configuration();

126. FileSystem hdfs = FileSystem.get(URI.create(uri), config);

127. Path src = new Path(localFile);

128. Path dst = new Path(hdfsFile);

129. hdfs.copyFromLocalFile(src, dst);

130. hdfs.close();

131. return true;

132. }

133.  

134. /*

135. * create a new file in the hdfs.

136. *

137. * notice that the toCreateFilePath is the full path

138. *

139. * and write the content to the hdfs file.

140. */

141. /**

142. * create a new file in the hdfs.

143. * if dir not exists, it will create one

144. *

145. * @param newFile new file path, a full path name, may like '/tmp/test.txt'

146. * @param content file content

147. * @return boolean true-success, false-failed

148. * @throws IOException file io exception

149. */

150. public static boolean createNewHDFSFile(String newFile, String content) throws IOException {

151. if (StringUtils.isBlank(newFile) || null == content) {

152. return false;

153. }

154. newFile = uri + newFile;

155. Configuration config = new Configuration();

156. FileSystem hdfs = FileSystem.get(URI.create(newFile), config);

157. FSDataOutputStream os = hdfs.create(new Path(newFile));

158. os.write(content.getBytes("UTF-8"));

159. os.close();

160. hdfs.close();

161. return true;

162. }

163.  

164. /**

165. * delete the hdfs file

166. *

167. * @param hdfsFile a full path name, may like '/tmp/test.txt'

168. * @return boolean true-success, false-failed

169. * @throws IOException file io exception

170. */

171. public static boolean deleteHDFSFile(String hdfsFile) throws IOException {

172. if (StringUtils.isBlank(hdfsFile)) {

173. return false;

174. }

175. hdfsFile = uri + hdfsFile;

176. Configuration config = new Configuration();

177. FileSystem hdfs = FileSystem.get(URI.create(hdfsFile), config);

178. Path path = new Path(hdfsFile);

179. boolean isDeleted = hdfs.delete(path, true);

180. hdfs.close();

181. return isDeleted;

182. }

183.  

184. /**

185. * read the hdfs file content

186. *

187. * @param hdfsFile a full path name, may like '/tmp/test.txt'

188. * @return byte[] file content

189. * @throws IOException file io exception

190. */

191. public static byte[] readHDFSFile(String hdfsFile) throws Exception {

192. if (StringUtils.isBlank(hdfsFile)) {

193. return null;

194. }

195. hdfsFile = uri + hdfsFile;

196. Configuration conf = new Configuration();

197. FileSystem fs = FileSystem.get(URI.create(hdfsFile), conf);

198. // check if the file exists

199. Path path = new Path(hdfsFile);

200. if (fs.exists(path)) {

201. FSDataInputStream is = fs.open(path);

202. // get the file info to create the buffer

203. FileStatus stat = fs.getFileStatus(path);

204. // create the buffer

205. byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];

206. is.readFully(0, buffer);

207. is.close();

208. fs.close();

209. return buffer;

210. } else {

211. throw new Exception("the file is not found .");

212. }

213. }

214.  

215. /**

216. * append something to file dst

217. *

218. * @param hdfsFile a full path name, may like '/tmp/test.txt'

219. * @param content string

220. * @return boolean true-success, false-failed

221. * @throws Exception something wrong

222. */

223. public static boolean append(String hdfsFile, String content) throws Exception {

224. if (StringUtils.isBlank(hdfsFile)) {

225. return false;

226. }

227. if(StringUtils.isEmpty(content)){

228. return true;

229. }

230.  

231. hdfsFile = uri + hdfsFile;

232. Configuration conf = new Configuration();

233. // solve the problem when appending at single datanode hadoop env

234. conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");

235. conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");

236. FileSystem fs = FileSystem.get(URI.create(hdfsFile), conf);

237. // check if the file exists

238. Path path = new Path(hdfsFile);

239. if (fs.exists(path)) {

240. try {

241. InputStream in = new ByteArrayInputStream(content.getBytes());

242. OutputStream out = fs.append(new Path(hdfsFile));

243. IOUtils.copyBytes(in, out, 4096, true);

244. out.close();

245. in.close();

246. fs.close();

247. } catch (Exception ex) {

248. fs.close();

249. throw ex;

250. }

251. } else {

252. HdfsUtils.createNewHDFSFile(hdfsFile, content);

253. }

254. return true;

255. }

256.  

257. }

 

HdfsUtilsTest .java

1. package test.com.xy6.demo.utils;

2.  

3. import static org.junit.Assert.assertEquals;

4.  

5. import java.util.ArrayList;

6. import java.util.List;

7.  

8. import org.junit.Test;

9.  

10. import com.xy6.demo.utils.HdfsUtils;

11.  

12. public class HdfsUtilsTest {

13.  

14. public static String uri = "hdfs://192.168.1.118:9000";

15. public String dir = "/user/output1";

16. public String parentDir = "/user";

17.  

18. @Test

19. public void testMkdirNull1() {

20. try{

21. assertEquals(false, HdfsUtils.mkdir(null));

22. assertEquals(false, HdfsUtils.mkdir(" "));

23. assertEquals(false, HdfsUtils.mkdir(""));

24. } catch(Exception ex){

25. assertEquals(true, false);

26. }

27. }

28.  

29. @Test

30. public void testMkdirNormal1() {

31. try{

32. HdfsUtils.deleteDir(dir);

33. boolean result = HdfsUtils.mkdir(dir);

34. assertEquals(true, result);

35.  

36. List<String> listFile = HdfsUtils.listAll(parentDir);

37. boolean existFile = false;

38. for(String elem : listFile){

39. if(elem.equals(uri + dir)){

40. existFile = true;

41. break;

42. }

43. }

44. assertEquals(true, existFile);

45. } catch(Exception ex){

46. ex.printStackTrace();

47. assertEquals(true, false);

48. }

49. }

50.  

51. @Test

52. public void testDeleteDirNull1() {

53. try{

54. assertEquals(false, HdfsUtils.deleteDir(null));

55. assertEquals(false, HdfsUtils.deleteDir(""));

56. assertEquals(false, HdfsUtils.deleteDir(" "));

57. } catch(Exception ex){

58. assertEquals(true, false);

59. }

60. }

61.  

62. @Test

63. public void testDeleteDir() {

64. try{

65. assertEquals(true, HdfsUtils.mkdir(dir));

66. assertEquals(true, HdfsUtils.deleteDir(dir));

67. List<String> listFile = HdfsUtils.listAll(parentDir);

68. boolean existFile = false;

69. for(String elem : listFile){

70. if(uri + dir == elem){

71. existFile = true;

72. break;

73. }

74. }

75. assertEquals(false, existFile);

76. } catch(Exception ex){

77. ex.printStackTrace();

78. assertEquals(true, false);

79. }

80. }

81.  

82. @Test

83. public void testListAllNull1() {

84. try{

85. List<String> listFile = new ArrayList<String>();

86. assertEquals(listFile.toString(), HdfsUtils.listAll(null).toString());

87. assertEquals(listFile.toString(), HdfsUtils.listAll(" ").toString());

88. assertEquals(listFile.toString(), HdfsUtils.listAll("").toString());

89. } catch(Exception ex){

90. assertEquals(true, false);

91. }

92. }

93.  

94. @Test

95. public void testListAllEmptyFolder() {

96. try{

97. HdfsUtils.deleteDir(dir);

98. assertEquals(true, HdfsUtils.mkdir(dir));

99. List<String> listFile = HdfsUtils.listAll(dir);

100. assertEquals(0, listFile.size());

101. } catch(Exception ex){

102. ex.printStackTrace();

103. assertEquals(true, false);

104. }

105. }

106.  

107. @Test

108. public void testListAllNotExistFolder() {

109. try{

110. HdfsUtils.deleteDir(dir);

111. List<String> listFile = HdfsUtils.listAll(dir);

112. assertEquals(0, listFile.size());

113. } catch(Exception ex){

114. assertEquals(true, true);

115. }

116. }

117.  

118. @Test

119. public void testUploadLocalFile2HDFSNull1() {

120. try{

121. assertEquals(false, HdfsUtils.uploadLocalFile2HDFS(null, null));

122. assertEquals(false, HdfsUtils.uploadLocalFile2HDFS("", ""));

123. assertEquals(false, HdfsUtils.uploadLocalFile2HDFS(" ", " "));

124. } catch(Exception ex){

125. assertEquals(true, false);

126. }

127. }

128.  

129. @Test

130. public void testUploadLocalFile2HDFS() {

131. String localFile = "F:/Program Files/eclipse/eclipse.ini";

132. String remoteFile = dir + "/eclipse.ini";

133.  

134. try{

135. HdfsUtils.mkdir(dir);

136. HdfsUtils.deleteHDFSFile(remoteFile);

137. assertEquals(true, HdfsUtils.uploadLocalFile2HDFS(localFile, remoteFile));

138. } catch(Exception ex){

139. ex.printStackTrace();

140. assertEquals(true, false);

141. }

142. }

143.  

144. @Test

145. public void testUploadLocalFile2HDFSLocalNotExist() {

146. String localFile = "F:/Program Files/eclipse/eclipse2.ini";

147. String remoteFile = dir + "/eclipse.ini";

148.  

149. try{

150. assertEquals(true, HdfsUtils.mkdir(dir));

151. HdfsUtils.deleteHDFSFile(remoteFile);

152. HdfsUtils.uploadLocalFile2HDFS(localFile, remoteFile);

153. } catch(Exception ex){

154. assertEquals(true, true);

155. }

156. }

157.  

158. @Test

159. public void testCreateNewHDFSFileNull1() {

160. try{

161. assertEquals(false, HdfsUtils.createNewHDFSFile(null, null));

162. assertEquals(false, HdfsUtils.createNewHDFSFile(" ", " "));

163. assertEquals(false, HdfsUtils.createNewHDFSFile("", ""));

164. } catch(Exception ex){

165. assertEquals(true, false);

166. }

167. }

168.  

169. @Test

170. public void testCreateNewHDFSFileNormal1() {

171. try{

172. String newFile = dir + "/file1.txt";

173. String content = "hello file1";

174.  

175. HdfsUtils.deleteHDFSFile(newFile);

176. assertEquals(true, HdfsUtils.createNewHDFSFile(newFile, content));

177. String result = new String(HdfsUtils.readHDFSFile(newFile));

178. assertEquals(content, result);

179. } catch(Exception ex){

180. ex.printStackTrace();

181. assertEquals(true, false);

182. }

183. }

184.  

185. @Test

186. public void testCreateNewHDFSFileFoldNotexist1() {

187. try{

188. String newFile = dir + "/file1.txt";

189. String content = "hello file1";

190.  

191. assertEquals(true, HdfsUtils.deleteDir(dir));

192. assertEquals(true, HdfsUtils.createNewHDFSFile(newFile, content));

193. } catch(Exception ex){

194. ex.printStackTrace();

195. assertEquals(true, false);

196. }

197. }

198.  

199. @Test

200. public void testDeleteHDFSFileNull1() {

201. try{

202. assertEquals(false, HdfsUtils.deleteHDFSFile(null));

203. assertEquals(false, HdfsUtils.deleteHDFSFile(" "));

204. assertEquals(false, HdfsUtils.deleteHDFSFile(""));

205. } catch(Exception ex){

206. assertEquals(true, false);

207. }

208. }

209.  

210. @Test

211. public void testDeleteHDFSFile() {

212. this.testUploadLocalFile2HDFS();

213. try{

214. String remoteFile = dir + "/eclipse.ini";

215. assertEquals(true, HdfsUtils.deleteHDFSFile(remoteFile));

216. } catch(Exception ex){

217. assertEquals(true, false);

218. }

219. }

220.  

221. @Test

222. public void testDeleteHDFSFileNotexist1() {

223. try{

224. String remoteFile = dir + "/eclipse2.ini";

225. assertEquals(false, HdfsUtils.deleteHDFSFile(remoteFile));

226. } catch(Exception ex){

227. assertEquals(true, false);

228. }

229. }

230.  

231. @Test

232. public void testReadHDFSFileNull1() {

233. try{

234. assertEquals(null, HdfsUtils.readHDFSFile(null));

235. assertEquals(null, HdfsUtils.readHDFSFile(" "));

236. assertEquals(null, HdfsUtils.readHDFSFile(""));

237. } catch(Exception ex){

238. assertEquals(true, false);

239. }

240. }

241.  

242. @Test

243. public void testReadHDFSFile() {

244. this.testUploadLocalFile2HDFS();

245. try{

246. String remoteFile = dir + "/eclipse.ini";

247. String result = new String(HdfsUtils.readHDFSFile(remoteFile));

248. assertEquals(true, result.length() > 0);

249. } catch(Exception ex){

250. ex.printStackTrace();

251. assertEquals(true, false);

252. }

253. }

254.  

255. @Test

256. public void testAppendNull1() {

257. try{

258. assertEquals(false, HdfsUtils.append(null, null));

259. assertEquals(false, HdfsUtils.append(" ", " "));

260. assertEquals(false, HdfsUtils.append("", ""));

261. } catch(Exception ex){

262. assertEquals(true, false);

263. }

264. }

265.  

266. @Test

267. public void testAppend() {

268. try{

269. String newFile = dir + "/file1.txt";

270. String content1 = "hello append1\r\n";

271. String content2 = "hello append2\r\n";

272.  

273. HdfsUtils.deleteHDFSFile(newFile);

274. assertEquals(true, HdfsUtils.createNewHDFSFile(newFile, ""));

275. assertEquals(true, HdfsUtils.append(newFile, content1));

276. assertEquals(content1, new String(HdfsUtils.readHDFSFile(newFile)));

277. assertEquals(true, HdfsUtils.append(newFile, content2));

278. assertEquals(content1 + content2, new String(HdfsUtils.readHDFSFile(newFile)));

279. } catch(Exception ex){

280. ex.printStackTrace();

281. assertEquals(true, false);

282. }

283. }

284.  

285. }