系统环境：centos7
软件版本：jdk1.8、zookeeper3.4.8、hadoop2.8.5
本次实验使用版本 kafka_2.12-3.0.0
一、安装
Kafka官网
将安装包上传至linux服务器上
解压
tar -zxvf kafka_2.12-3.0.0.tgz -C /home/local/
移动目录至kafka
mv kafka_2.12-3.0.0/ kafka
二、部署
配置Kafka环境
vi /etc/profile
添加如下配置
#kafka
export KAFKA_HOME=/home/local/kafka
export PATH=$PATH:${KAFKA_HOME}/bin
修改server.properties文件
vim /home/local/kafka/config/server.properties
修改参数如下：
broker.id=0
listeners=PLAINTEXT://192.168.245.200:9092
log.dirs=/tmp/kafka-logs
zookeeper.connect=192.168.245.200:2181,192.168.245.201:2181,192.168.245.202:2181
参数说明：
broker.id ： 集群内全局唯一标识，每个节点上需要设置不同的值
listeners：这个IP地址也是与本机相关的，每个节点上设置为自己的IP地址
log.dirs ：存放kafka消息的
zookeeper.connect ： 配置的是zookeeper集群地址
分发kafka安装目录
for i in {1..2};do scp -r /home/local/kafka root@slave${i}:/home/local/;done
三、启动
进入kafka安装目录下
./bin/kafka-server-start.sh ./config/server.properties &
kafka相关命令
创建topic
kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test显示所有topic
kafka-topics.sh --list --bootstrap-server localhost:9092产生消息
kafka-console-producer.sh --broker-list localhost:9092 --topic test消费消息
kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning删除topic
kafka-topics.sh --delete --bootstrap-server localhost:9092 --topic test四、flink与kafka结合示例
首先 ，构建maven工程，加入flink与kafka的一些依赖：
<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>org.example</groupId><artifactId>bigdata-kafka_2.12-3.0.0</artifactId><version>1.0-SNAPSHOT</version><name>bigdata-kafka_2.12-3.0.0</name><!-- FIXME change it to the project's website --><url>http://www.example.com</url><properties><project.build.sourceEncoding>UTF-8</project.build.sourceEncoding><maven.compiler.source>1.8</maven.compiler.source><maven.compiler.target>1.8</maven.compiler.target><flink-version>1.14.0</flink-version><scala.binary.version>2.11.2</scala.binary.version></properties><dependencies><dependency><groupId>org.apache.flink</groupId><artifactId>flink-java</artifactId><version>${flink-version}</version></dependency><dependency><groupId>org.apache.flink</groupId><artifactId>flink-streaming-java_2.11</artifactId><version>${flink-version}</version></dependency><dependency><groupId>org.apache.flink</groupId><artifactId>flink-clients_2.11</artifactId><version>${flink-version}</version></dependency><dependency><groupId>org.apache.flink</groupId><artifactId>flink-connector-kafka_2.11</artifactId><version>${flink-version}</version></dependency><dependency><groupId>junit</groupId><artifactId>junit</artifactId><version>4.11</version><scope>test</scope></dependency></dependencies>
</project>第一个，flink生产者示例代码：
package com.example;import org.apache.commons.lang3.RandomStringUtils;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;import java.io.Serializable;
import java.util.Properties;public class KafkaProducerExample {public static void main(String[] args) throws Exception {final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();Properties props = new Properties();props.setProperty("bootstrap.servers", "192.168.245.200:9092");DataStream<String> stream = env.addSource(new SimpleStringGenerator());stream.addSink(new FlinkKafkaProducer<String>("test", new SimpleStringSchema(), props));env.execute();}
}class SimpleStringGenerator implements SourceFunction<String>, Serializable {private static final long serialVersionUID = 1L;private volatile boolean isRunning = true;@Overridepublic void run(SourceContext<String> ctx) throws Exception {while (isRunning) {String str = RandomStringUtils.randomAlphanumeric(5);ctx.collect(str);Thread.sleep(1000);}}@Overridepublic void cancel() {isRunning = false;}
}
因为flink是生产者，需要启动一个kafka的消费者终端，然后运行本示例：
启动kafka
bin/kafka-server-start.sh config/server.properties &
启动一个kafka的消费者终端
bin/kafka-console-consumer.sh --bootstrap-server master:9092 --topic test
终端内容
第二个，flink消费者示例代码：
package com.example;import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;import java.util.Properties;public class KafkaConsumerApp {public static void main(String[] args) {try {final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();Properties properties = new Properties();properties.setProperty("bootstrap.servers", "master:9092");properties.setProperty("group.id", "flink");DataStream<String> stream = env.addSource(new FlinkKafkaConsumer<String>("test", new SimpleStringSchema(), properties));stream.map(new MapFunction<String, Object>() {@Overridepublic Object map(String value) throws Exception {return "flink: " + value;}}).print();env.execute("consumer");} catch (Exception e) {e.printStackTrace();}}
}为了测试，我们先开启一个生产者，不断往kafka中发送消息。
kafka-console-producer.sh --broker-list master:9092 --topic test
终端
控制台
打印结果符合预期，flink与kafka结合的示例就演示完成了，主要的还是熟悉flink编程。
