本文介绍了ASP.NET实现Hadoop增删改查的示例代码，分享给大家，具体如下：
packages.config
<?xml version="1.0" encoding="utf-8"?><packages> <package id="Microsoft.AspNet.WebApi.Client" version="4.0.20505.0" targetFramework="net46" /> <package id="Microsoft.Data.Edm" version="5.2.0" targetFramework="net46" /> <package id="Microsoft.Data.OData" version="5.2.0" targetFramework="net46" /> <package id="Microsoft.Hadoop.WebClient" version="0.12.5126.42915" targetFramework="net46" /> <package id="Microsoft.Net.Http" version="2.0.20505.0" targetFramework="net46" /> <package id="Microsoft.WindowsAzure.ConfigurationManager" version="1.8.0.0" targetFramework="net46" /> <package id="Newtonsoft.Json" version="4.5.11" targetFramework="net46" /> <package id="System.Spatial" version="5.2.0" targetFramework="net46" /> <package id="WindowsAzure.Storage" version="2.0.4.1" targetFramework="net46" /></packages>
HDFSAccess.cs
using Microsoft.Hadoop.WebHDFS;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
namespace Physical{
public sealed class HDFSAccess {
private readonly WebHDFSClient webHDFSClient;
public HDFSAccess(string uriString, string userName) {
this.webHDFSClient = new WebHDFSClient(new Uri(uriString), userName);
}
public List<string> GetDirectories(string path) {
var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;
return directoryStatus.Directories.Select(d => d.PathSuffix).ToList();
}
public List<string> GetFiles(string path) {
var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;
return directoryStatus.Files.Select(d => d.PathSuffix).ToList();
}
public bool CreateDirectory(string path) {
// 传入路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.CreateDirectory(path).Result;
}
public bool DeleteDirectory(string path) {
// 传入路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.DeleteDirectory(path).Result;
}
public string CreateFile(string localFile, string remotePath) {
// 传入远端路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.CreateFile(localFile, remotePath).Result;
}
public bool DeleteFile(string path) {
// 传入路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.DeleteDirectory(path).Result;
}
public HttpResponseMessage OpenFile(string path) {
// 传入路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.OpenFile(path).Result;
}
}
}
Program.cs
using Physical;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace MyTest{
class Program {
// HDFS cluster 客户端进入端点设定于 主机上 // 预设端点：http://[主机名称]:50070 // 预设帐号：Wu private static HDFSAccess access = new HDFSAccess(@"http://127.0.0.1:50070", "Wu");
static void Main(string[] args) {
GetDirectoriesTest();
Console.WriteLine("----------------------------------------------");
GetFilesTest();
Console.WriteLine("----------------------------------------------");
DirectoryTest();
Console.WriteLine("----------------------------------------------");
FileTest();
Console.WriteLine("----------------------------------------------");
OpenFileTest();
Console.ReadKey();
}
public void TestCleanup() {
//取得根目录资料夹 var directories = access.GetDirectories(@"/");
// 移除预设目录：tmp、user 外的目录 foreach (var directory in directories) {
if ("tmp".Equals(directory) || "user".Equals(directory)) {
continue;
}
else {
access.DeleteDirectory(directory);
}
}
// 取得根目录档案 var files = access.GetFiles(@"/");
// 移除所有档案 foreach (var file in files) {
access.DeleteFile(file);
}
// 移除 OpenFile 转存档案 File.Delete(Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg"));
}
//_传入根目录_预期回传预设目录 public static void GetDirectoriesTest() {
// 预设根目录下有两个目录：tmp、user var expected = new List<string>() {
"tmp", "user", }
;
var actual = access.GetDirectories(@"/");
foreach (var item in actual) {
Console.WriteLine(item);
}
}
//_传入根目录_预期回传空集合 public static void GetFilesTest() {
// 预设根目录下没有档案 var expected = new List<string>();
var actual = access.GetFiles(@"/");
foreach (var item in actual) {
Console.WriteLine(item);
}
}
//_建立zzz目录_预期成功_预期根目录下有zzz目录_删除zzz目录_预期成功_预期根目录下无zzz目录 public static void DirectoryTest() {
var directoryName = "zzz";
// 建立zzz目录 var boolCreateDirectory = access.CreateDirectory(directoryName);
Console.WriteLine("建立zzz目录_预期成功:"+boolCreateDirectory);
// 建立zzz目录_预期成功_预期根目录下有zzz目录 // 预设根目录下有三个目录：tmp、user、zzz var expectedCreateDirectory = new List<string>() {
"tmp", "user", directoryName, }
;
var actualCreateDirectory = access.GetDirectories(@"/");
foreach (var item in actualCreateDirectory) {
Console.WriteLine(item);
}
Console.WriteLine("********************************************");
// 删除zzz目录 var boolDeleteDirectory = access.DeleteDirectory(directoryName);
Console.WriteLine("删除zzz目录_预期成功:" + boolDeleteDirectory);
// 删除zzz目录_预期成功_预期根目录下无zzz目录 // 预设根目录下有两个目录：tmp、user var expectedDeleteDirectory = new List<string>() {
"tmp", "user", }
;
var actualDeleteDirectory = access.GetDirectories(@"/");
foreach (var item in actualDeleteDirectory) {
Console.WriteLine(item);
}
}
//_建立Test档案_预期根目录下有Test档案_删除Test档案_预期成功_预期根目录下无Test档案 public static void FileTest() {
var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");
var remotePath = "Test.jpg";
// 建立Test档案 var boolCreateFile = access.CreateFile(localFile, remotePath);
// 建立Test档案_预期根目录下有Test档案 var expectedCreateFile = new List<string>() {
remotePath, }
;
var actualCreateFile = access.GetFiles(@"/");
foreach (var item in actualCreateFile) {
Console.WriteLine(item);
}
Console.WriteLine("********************************************");
// 删除Test档案 var boolDeleteFile = access.DeleteDirectory(remotePath);
Console.WriteLine("删除Test档案_预期成功:"+boolDeleteFile);
// 删除Test档案_预期成功_预期根目录下无Test档案 var expectedDeleteFile = new List<string>();
var actualDeleteFile = access.GetFiles(@"/");
foreach (var item in actualDeleteFile) {
Console.WriteLine(item);
}
}
//_建立Test档案_预期根目录下有Test档案_取得Test档案_预期成功_预期回传Test档案Stream并转存成功 public static void OpenFileTest() {
var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");
var remotePath = "Test.jpg";
var saveFile = Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg");
Console.WriteLine("saveFile:" + saveFile);
Console.WriteLine("********************************************");
// 建立Test档案 var boolCreateFile = access.CreateFile(localFile, remotePath);
// 建立Test档案_预期根目录下有Test档案 var expectedCreateFile = new List<string>() {
remotePath, }
;
var actualCreateFile = access.GetFiles(@"/");
foreach (var item in actualCreateFile) {
Console.WriteLine(item);
}
Console.WriteLine("********************************************");
// 取得Test档案 var response = access.OpenFile(remotePath);
// 取得Test档案_预期成功 response.EnsureSuccessStatusCode();
// 取得Test档案_预期成功_预期回传Test档案Stream并转存成功 using (var fs = File.Create(saveFile)) {
response.Content.CopyToAsync(fs).Wait();
}
Console.WriteLine(File.Exists(saveFile));
}
}
}
运行结果如图:
以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持脚本之家。
您可能感兴趣的文章:
- hadoop动态增加和删除节点方法介绍
- Hadoop编程基于MR程序实现倒排索引示例
- Hadoop对文本文件的快速全局排序实现方法及分析
- Hadoop 中 HBase Shell命令的详解
- docker 搭建hadoop以及hbase集群详解
- 浅谈七种常见的Hadoop和Spark项目案例
- hadoop格式化HDFS出现错误解决办法
- VMware虚拟机下hadoop1.x的安装方法
- Hadoop上Data Locality的详解
