一、需求
依然以wordcount
案例为基础,进行开发
我们知道,用hadoop
自带的example.jar
执行wordcount
命令如下
hadoop jar /opt/module/hadoop-3.1.3/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.3.jar wordcount -D mapreduce.job.queuename=hive /input /output1
这个命令中, 首先,指定方法名参数,然后,指定了队列-D参数,最后,指定输入文件和输出目录。
那么我们自己开发的wc.jar可以这样用吗?
二、测试之前的wc.jar
命令1
hadoop jar wc.jar com.atguigu.mapreduce.wordcount.WordCountDriver wordcount -D mapreduce.job.queuename=hive /input /output1
报错
命令2
hadoop jar wc.jar com.atguigu.mapreduce.wordcount.WordCountDriver -D mapreduce.job.queuename=hive /input /output1
报错
可以发现,我们自己开发的jar,不能,像hadoop自带的jar一样,带上其他各种参数。
三、实现Tool接口的jar
WordCount
package com.atguigu.mapreduce.tool;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;import java.io.IOException;public class WordCount implements Tool {private Configuration conf;// 核心驱动(conf 需要传入)@Overridepublic int run(String[] args) throws Exception {Job job = Job.getInstance(conf);job.setJarByClass(WordCountDriver.class);job.setMapperClass(WordCountMapper.class);job.setReducerClass(WordCountReducer.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);FileInputFormat.setInputPaths(job, new Path(args[0]));FileOutputFormat.setOutputPath(job, new Path(args[1]));return job.waitForCompletion(true) ? 0 : 1;}@Overridepublic void setConf(Configuration conf) {this.conf = conf;}@Overridepublic Configuration getConf() {return conf;}// mapperpublic static class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {private Text outK = new Text();private IntWritable outV = new IntWritable(1);@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {// ss cls// 1 获取一行String line = value.toString();// 2 切割String[] words = line.split(" ");// 3 循环遍历写出for (String word : words) {outK.set(word);context.write(outK, outV);}}}// reducerpublic static class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {private IntWritable outV = new IntWritable();@Overrideprotected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {int sum = 0;for (IntWritable value : values) {sum += value.get();}outV.set(sum);context.write(key, outV);}}
}
WordCountDriver
package com.atguigu.mapreduce.tool;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;import java.util.Arrays;public class WordCountDriver {private static Tool tool;public static void main(String[] args) throws Exception {// 创建配置Configuration conf = new Configuration();switch (args[0]){case "wordcount":tool = new WordCount();break;default:throw new RuntimeException("no such tool "+ args[0]);}// 执行程序int run = ToolRunner.run(conf, tool, Arrays.copyOfRange(args, 1, args.length));System.exit(run);}
}
测试命令
hadoop jar wc.jar com.atguigu.mapreduce.tool.WordCountDriver wordcount -D mapreduce.job.queuename=hive /input /output1