rm -rf /opt/linuxsir/hadoop/logs/*.*
ssh root@192.168.31.132 rm -rf /opt/linuxsir/hadoop/logs/*.*
ssh root@192.168.31.133 rm -rf /opt/linuxsir/hadoop/logs/*.*
clear
cd /opt/linuxsir/hadoop/sbin
./start-dfs.sh
./start-yarn.sh
clear
jps
ssh root@192.168.31.132 jps
ssh root@192.168.31.133 jps
在eclipse里面操作如下:
New
-Java Project
,名称自定义即可,如 java-prj
New
-Package
,名称自定义为com.pai.hdfs_demo
New
-Class
,名称自定义为ReadWriteHDFSExample
package com.pai.hdfs_demo;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.*;
import java.nio.charset.StandardCharsets;
public class ReadWriteHDFSExample {
// main 新建一个类ReadWriteHDFSExample,编写main函数如下。main函数调用其它函数,创建目录,写入数据,添加数据,然后再读取数据
public static void main(String[] args) throws IOException {
// ReadWriteHDFSExample.checkExists();
ReadWriteHDFSExample.createDirectory();
ReadWriteHDFSExample.writeFileToHDFS();
ReadWriteHDFSExample.appendToHDFSFile();
ReadWriteHDFSExample.readFileFromHDFS();
}
// readFileFromHDFS 该函数读取文件内容,以字符串形式显示出来
public static void readFileFromHDFS() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", "hdfs://192.168.31.131:9000");
FileSystem fileSystem = FileSystem.get(configuration);
// Create a path
String fileName = "read_write_hdfs_example.txt";
Path hdfsReadPath = new Path("/javareadwriteexample/" + fileName);
// initialize input stream
FSDataInputStream inputStream = fileSystem.open(hdfsReadPath);
// Classical input stream usage
String out = IOUtils.toString(inputStream, "UTF-8");
System.out.println(out);
// BufferedReader bufferedReader = new BufferedReader(
// new InputStreamReader(inputStream, StandardCharsets.UTF_8));
// String line = null;
// while ((line=bufferedReader.readLine())!=null){
// System.out.println(line);
// }
inputStream.close();
fileSystem.close();
}
// writeFileToHDFS writeFileToHDFS函数打开文件,写入一行文本
public static void writeFileToHDFS() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", "hdfs://192.168.31.131:9000");
FileSystem fileSystem = FileSystem.get(configuration);
// Create a path
String fileName = "read_write_hdfs_example.txt";
Path hdfsWritePath = new Path("/javareadwriteexample/" + fileName);
FSDataOutputStream fsDataOutputStream = fileSystem.create(hdfsWritePath, true);
BufferedWriter bufferedWriter = new BufferedWriter(
new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8));
bufferedWriter.write("Java API to write data in HDFS");
bufferedWriter.newLine();
bufferedWriter.close();
fileSystem.close();
}
// appendToHDFSFile 函数打开文件,添加一行文本。需要注意的是,需要对Configuration类的对象configuration进行适当设置,否则出错
public static void appendToHDFSFile() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", "hdfs://192.168.31.131:9000");
//configuration.setBoolean("dfs.client.block.write.replace-datanode-on-failure.enabled", true);
configuration.set("dfs.client.block.write.replace-datanode-on-failure.policy","NEVER");
configuration.set("dfs.client.block.write.replace-datanode-on-failure.enable","true");
FileSystem fileSystem = FileSystem.get(configuration);
// Create a path
String fileName = "read_write_hdfs_example.txt";
Path hdfsWritePath = new Path("/javareadwriteexample/" + fileName);
FSDataOutputStream fsDataOutputStream = fileSystem.append(hdfsWritePath);
BufferedWriter bufferedWriter = new BufferedWriter(
new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8));
bufferedWriter.write("Java API to append data in HDFS file");
bufferedWriter.newLine();
bufferedWriter.close();
fileSystem.close();
}
// createDirectory 函数创建一个目录
public static void createDirectory() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", "hdfs://192.168.31.131:9000");
FileSystem fileSystem = FileSystem.get(configuration);
String directoryName = "/javareadwriteexample";
Path path = new Path(directoryName);
fileSystem.mkdirs(path);
}
// checkExists checkExists检查目录或者文件是否存在。注意如下代码的最后一个括号是ReadWriteHDFSExample类的结束括号
public static void checkExists() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", "hdfs://192.168.31.131:9000");
FileSystem fileSystem = FileSystem.get(configuration);
String directoryName = "/javareadwriteexample";
Path path = new Path(directoryName);
if (fileSystem.exists(path)) {
System.out.println("File/Folder Exists : " + path.getName());
} else {
System.out.println("File/Folder does not Exists : " + path.getName());
}
}
}
为了编译通过上述Java代码,需要把如下目录下的jar包导入Eclipse项目的Build Path
操作序列为 右键点击Eclipse里的Java项目→Properties
→Java Build Path
→Libraries
→Add External Jars
# 添加如下路径的包
D:hadoop-2.7.3sharehadoopcommonlib
D:hadoop-2.7.3sharehadoopcommon
D:hadoop-2.7.3sharehadoophdfs
D:hadoop-2.7.3sharehadoophdfslib
D:hadoop-2.7.3sharehadoopmapreducelib
D:hadoop-2.7.3sharehadoopmapreduce
D:hadoop-2.7.3sharehadoopyarnlib
D:hadoop-2.7.3sharehadoopyarn
就可以愉快地执行了,执行完毕上述代码后,在hd-master主机上可以通过如下命令,检查已经写入的文件
[root@hd-master bin]# cd /opt/linuxsir/hadoop/bin
[root@hd-master bin]# ./hdfs dfs -ls /javareadwriteexample/read_write_hdfs_example.txt
-rw-r--r-- 3 root supergroup 70 2024-10-10 04:47 /javareadwriteexample/read_write_hdfs_example.txt
[root@hd-master bin]# ./hdfs dfs -cat /javareadwriteexample/read_write_hdfs_example.txt
Java API to write data in HDFS
Java API to append data in HDFS file
为了多次进行实验(或者为了调试代码),可以把HDFS文件删除,然后再执行或者调试Java代码,否则一经存在该目录,执行创建目录的代码就会出错
cd /opt/linuxsir/hadoop/bin
./hdfs dfs -rm /javareadwriteexample/*
./hdfs dfs -rmdir /javareadwriteexample
cd /opt/linuxsir/hadoop/sbin
./stop-yarn.sh
./stop-dfs.sh
jps
ssh root@192.168.31.132 jps
ssh root@192.168.31.133 jps
package mywordcount;
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
public class WordCount {
//定义WordCount类的内部类TokenizerMapper 该类实现了map函数,把从文件读取的每个word变成一个形式为<word,1>的Key Value对,输出到map函数的参数context对象,由执行引擎完成Shuffle
public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
context.write(word, one);
}
}
}
//定义WordCount类的内部类IntSumReducer 该类实现了reduce函数,它收拢所有相同key的、形式为<word,1>的Key-Value对,对Value部分进行累加,输出一个计数
public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
String thekey = key.toString();
int thevalue = sum;
}
}
// WordCount类的main函数,负责配置Job的若干关键的参数,并且启动这个Job。在main函数中,conf对象包含了一个属性即“fs.defaultFS”,它的值为“hdfs://192.168.31.131:9000”,使得WordCount程序知道如何存取HDFS
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: wordcount <in> <out>");
System.exit(2);
}
conf.set("fs.defaultFS", "hdfs://192.168.31.131:9000");
Job job = new Job(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
[root@hd-master bin]# ./hdfs dfs -ls /output1
Found 2 items
-rw-r--r-- 3 root supergroup 0 2024-10-10 05:17 /output1/_SUCCESS
-rw-r--r-- 3 root supergroup 89 2024-10-10 05:17 /output1/part-r-00000
[root@hd-master bin]# ./hdfs dfs -cat /output1/part-r-00000
I 1
apache 1
cloudera 1
google 1
hadoop 8
hortonworks 1
ibm 1
intel 1
like 1
microsoft 1
数据分析咨询请扫描二维码
若不方便扫码,搜微信号:CDAshujufenxi
你是否被统计学复杂的理论和晦涩的公式劝退过?别担心,“山有木兮:统计学极简入门(Python)” 将为你一一化解这些难题。课程 ...
2025-03-31在电商、零售、甚至内容付费业务中,你真的了解你的客户吗? 有些客户下了一两次单就消失了,有些人每个月都回购,有些人曾经是 ...
2025-03-31在数字化浪潮中,数据驱动决策已成为企业发展的核心竞争力,数据分析人才的需求持续飙升。世界经济论坛发布的《未来就业报告》, ...
2025-03-28你有没有遇到过这样的情况?流量进来了,转化率却不高,辛辛苦苦拉来的用户,最后大部分都悄无声息地离开了,这时候漏斗分析就非 ...
2025-03-27TensorFlow Datasets(TFDS)是一个用于下载、管理和预处理机器学习数据集的库。它提供了易于使用的API,允许用户从现有集合中 ...
2025-03-26"不谋全局者,不足谋一域。"在数据驱动的商业时代,战略级数据分析能力已成为职场核心竞争力。《CDA二级教材:商业策略数据分析 ...
2025-03-26当你在某宝刷到【猜你喜欢】时,当抖音精准推来你的梦中情猫时,当美团外卖弹窗刚好是你想吃的火锅店…… 恭喜你,你正在被用户 ...
2025-03-26当面试官问起随机森林时,他到底在考察什么? ""请解释随机森林的原理""——这是数据分析岗位面试中的经典问题。但你可能不知道 ...
2025-03-25在数字化浪潮席卷的当下,数据俨然成为企业的命脉,贯穿于业务运作的各个环节。从线上到线下,从平台的交易数据,到门店的运营 ...
2025-03-25在互联网和移动应用领域,DAU(日活跃用户数)是一个耳熟能详的指标。无论是产品经理、运营,还是数据分析师,DAU都是衡量产品 ...
2025-03-24ABtest做的好,产品优化效果差不了!可见ABtest在评估优化策略的效果方面地位还是很高的,那么如何在业务中应用ABtest? 结合企业 ...
2025-03-21在企业数据分析中,指标体系是至关重要的工具。不仅帮助企业统一数据标准、提升数据质量,还能为业务决策提供有力支持。本文将围 ...
2025-03-20解锁数据分析师高薪密码,CDA 脱产就业班助你逆袭! 在数字化浪潮中,数据驱动决策已成为企业发展的核心竞争力,数据分析人才的 ...
2025-03-19在 MySQL 数据库中,查询一张表但是不包含某个字段可以通过以下两种方法实现:使用 SELECT 子句以明确指定想要的字段,或者使 ...
2025-03-17在当今数字化时代,数据成为企业发展的关键驱动力,而用户画像作为数据分析的重要成果,改变了企业理解用户、开展业务的方式。无 ...
2025-03-172025年是智能体(AI Agent)的元年,大模型和智能体的发展比较迅猛。感觉年初的deepseek刚火没多久,这几天Manus又成为媒体头条 ...
2025-03-14以下的文章内容来源于柯家媛老师的专栏,如果您想阅读专栏《小白必备的数据思维课》,点击下方链接 https://edu.cda.cn/goods/sh ...
2025-03-13以下的文章内容来源于刘静老师的专栏,如果您想阅读专栏《10大业务分析模型突破业务瓶颈》,点击下方链接 https://edu.cda.cn/go ...
2025-03-12以下的文章内容来源于柯家媛老师的专栏,如果您想阅读专栏《小白必备的数据思维课》,点击下方链接 https://edu.cda.cn/goods/sh ...
2025-03-11随着数字化转型的加速,企业积累了海量数据,如何从这些数据中挖掘有价值的信息,成为企业提升竞争力的关键。CDA认证考试体系应 ...
2025-03-10