Kubernetes运行大数据组件-制作镜像

软件包

● jdk-8u221-linux-x64.tar.gz
● hadoop-2.10.1.tar.gz
● apache-hive-2.3.8-bin.tar.gz
● scala-2.12.13.tgz
● spark-3.0.3-bin-hadoop2.7.tgz

构建镜像

注意：需要提前解压软件包

java镜像

Dockerfile

FROM centos:7
RUN yum makecache && yum install net-tools lsof vim curl wget -y && rm /var/cache -rf
RUN echo -e "* hard nofile 102400\n* soft nofile 102400" >>/etc/security/limits.conf
COPY ./jdk1.8.0_221 /opt/jdk
ENV JAVA_HOME="/opt/jdk"
ENV JRE_HOME="$JAVA_HOME/jre"
ENV CLASSPATH=".:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib/rt.jar"
ENV TZ=Asia/Shanghai
ENV PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin

构建镜像

[root@docker jdk]# ls
Dockerfile  jdk1.8.0_221
[root@docker jdk]# docker build -t oraclejdk:8u221 .
Sending build context to Docker daemon    408MB
Step 1/9 : FROM centos:7---> 8652b9f0cb4c
Step 2/9 : RUN yum makecache && yum install net-tools lsof vim curl wget -y && rm /var/cache -rf---> Using cache---> 08bd1ebf0b99
Step 3/9 : RUN echo -e "* hard nofile 102400\n* soft nofile 102400" >>/etc/security/limits.conf---> Using cache---> e77c87d7751b
Step 4/9 : COPY ./jdk1.8.0_221 /opt/jdk---> 1c0ad086925f
Step 5/9 : ENV JAVA_HOME="/opt/jdk"---> Running in 1345a5af59ee
Removing intermediate container 1345a5af59ee---> 9a9947713905
Step 6/9 : ENV JRE_HOME="$JAVA_HOME/jre"---> Running in bfc8e29ada61
Removing intermediate container bfc8e29ada61---> 9ef0ca8afb5c
Step 7/9 : ENV CLASSPATH=".:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib/rt.jar"---> Running in df4e6c135a23
Removing intermediate container df4e6c135a23---> bdd0ede47222
Step 8/9 : ENV TZ=Asia/Shanghai---> Running in ed6b3ecc3b0d
Removing intermediate container ed6b3ecc3b0d---> e0b340c2f008
Step 9/9 : ENV PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin---> Running in 59bbf9e4003e
Removing intermediate container 59bbf9e4003e---> c5ec18592869
Successfully built c5ec18592869
Successfully tagged oraclejdk:8u221
[root@docker spark]#

验证镜像

[root@docker jdk]# docker run -it --rm oraclejdk:8u221 java -version
java version "1.8.0_221"
Java(TM) SE Runtime Environment (build 1.8.0_221-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.221-b11, mixed mode)
[root@docker spark]#

hadoop镜像

Dockerfile

FROM oraclejdk:8u221
COPY ./hadoop-2.10.1 /opt/hadoop
ENV HADOOP_HOME="/opt/hadoop"
ENV HADOOP_CONF_DIR="$HADOOP_HOME/etc/hadoop"
ENV HADOOP_LOG_DIR="$HADOOP_HOME/logs"
ENV YARN_CONF_DIR="$HADOOP_CONF_DIR"
ENV YARN_LOG_DIR="$HADOOP_LOG_DIR"
ENV PATH="$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin"

构建镜像

[root@docker hadoop]# ls
Dockerfile  hadoop-2.10.1
[root@docker hadoop]# docker build -t hadoop:2.10.1 .
Sending build context to Docker daemon  461.3MB
Step 1/8 : FROM oraclejdk:8u221---> c5ec18592869
Step 2/8 : COPY ./hadoop-2.10.1 /opt/hadoop---> a3e0be814121
Step 3/8 : ENV HADOOP_HOME="/opt/hadoop"---> Running in e6349d12c8d6
Removing intermediate container e6349d12c8d6---> 73ad5061fbf9
Step 4/8 : ENV HADOOP_CONF_DIR="$HADOOP_HOME/etc/hadoop"---> Running in 06c7168f4b7c
Removing intermediate container 06c7168f4b7c---> fb3700df305f
Step 5/8 : ENV HADOOP_LOG_DIR="$HADOOP_HOME/logs"---> Running in be2c841c0264
Removing intermediate container be2c841c0264---> 53619a2f4455
Step 6/8 : ENV YARN_CONF_DIR="$HADOOP_CONF_DIR"---> Running in 5ad7246fabcd
Removing intermediate container 5ad7246fabcd---> b3597d2a86b6
Step 7/8 : ENV YARN_LOG_DIR="$HADOOP_LOG_DIR"---> Running in bb3b29f986d4
Removing intermediate container bb3b29f986d4---> fb6ef5502501
Step 8/8 : ENV PATH="$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin"---> Running in 6904f5c731ee
Removing intermediate container 6904f5c731ee---> e127c07d158c
Successfully built e127c07d158c
Successfully tagged hadoop:2.10.1
[root@docker spark]#

验证镜像：

[root@docker hadoop]# docker run -it --rm hadoop:2.10 hdfs version
Hadoop 2.10.1
Subversion https://github.com/apache/hadoop -r 1827467c9a56f133025f28557bfc2c562d78e816
Compiled by centos on 2020-09-14T13:17Z
Compiled with protoc 2.5.0
From source with checksum 3114edef868f1f3824e7d0f68be03650
This command was run using /opt/hadoop/share/hadoop/common/hadoop-common-2.10.1.jar
[root@docker spark]#

Hive镜像

Dockerfile

FROM hadoop:2.10.1
COPY ./hive-2.3.8 /opt/hive
ENV HIVE_HOME="/opt/hive"
ENV HIVE_CONF_DIR="$HIVE_HOME/conf"
ENV HIVE_AUX_JARS_PATH="$HIVE_HOME/lib"
ENV PATH="$PATH:$HIVE_HOME/bin"

构建镜像

[root@docker hive]# ls
Dockerfile  hive-2.3.8
[root@docker hive]# docker build -t hive:2.3.8 .
Sending build context to Docker daemon  328.2MB
Step 1/6 : FROM hadoop:2.10.1---> e127c07d158c
Step 2/6 : COPY ./hive-2.3.8 /opt/hive---> 6ea2ee6843e1
Step 3/6 : ENV HIVE_HOME="/opt/hive"---> Running in 4997ab43bb11
Removing intermediate container 4997ab43bb11---> cd04772f661e
Step 4/6 : ENV HIVE_CONF_DIR="$HIVE_HOME/conf"---> Running in b58062ea41b1
Removing intermediate container b58062ea41b1---> 3624917c8da5
Step 5/6 : ENV HIVE_AUX_JARS_PATH="$HIVE_HOME/lib"---> Running in d3e2502f3337
Removing intermediate container d3e2502f3337---> 8e8051f0c8ce
Step 6/6 : ENV PATH="$PATH:$HIVE_HOME/bin"---> Running in ac9bed6d6b07
Removing intermediate container ac9bed6d6b07---> 5b3bfee1aa99
Successfully built 5b3bfee1aa99
Successfully tagged hive:2.3.8
[root@docker hive]#

验证镜像

[root@docker hive]# docker run -it --rm hive:2.3.8 hive --help
Usage ./hive <parameters> --service serviceName <service parameters>
Service List: beeline cleardanglingscratchdir cli hbaseimport hbaseschematool help hiveburninclient hiveserver2 hplsql jar lineage llap llapdumpllapstatus metastore metatool orcfiledump rcfilecat schemaTool version
Parameters parsed:--auxpath : Auxiliary jars--config : Hive configuration directory--service : Starts specific service/component. cli is default
Parameters used:HADOOP_HOME or HADOOP_PREFIX : Hadoop install directoryHIVE_OPT : Hive options
For help on a particular service:./hive --service serviceName --help
Debug help:  ./hive --debug --help
[root@docker spark]#

Spark镜像

Dockerfile

FROM hadoop:2.10.1
COPY ./scala-2.12.13 /opt/scala
COPY ./spark-3.0.3-bin-hadoop2.7 /opt/spark
ENV SCALA_HOME="/opt/scala"
ENV SPARK_HOME="/opt/spark"
ENV SPARK_CONF_DIR="$SPARK_HOME/conf"
ENV PATH="$PATH:$SCALA_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin"

构建镜像

[root@docker spark]# docker build -t spark:3.0.3 .
Sending build context to Docker daemon  268.1MB
Step 1/7 : FROM hadoop:2.10.1---> e127c07d158c
Step 2/7 : COPY ./scala-2.12.13 /opt/scala---> 5c86434d6f57
Step 3/7 : COPY ./spark-3.0.3-bin-hadoop2.7 /opt/spark---> 7dab7d209113
Step 4/7 : ENV SCALA_HOME="/opt/scala"---> Running in 1e03c34269eb
Removing intermediate container 1e03c34269eb---> 2f1bc529d29a
Step 5/7 : ENV SPARK_HOME="/opt/spark"---> Running in d3ea2fc0d6be
Removing intermediate container d3ea2fc0d6be---> bff04158ce16
Step 6/7 : ENV SPARK_CONF_DIR="$SPARK_HOME/conf"---> Running in a9f388819c46
Removing intermediate container a9f388819c46---> 6e87dcba9463
Step 7/7 : ENV PATH="$PATH:$SCALA_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin"---> Running in f8b2d9adc0cb
Removing intermediate container f8b2d9adc0cb---> d7f9e8cad67d
Successfully built d7f9e8cad67d
Successfully tagged spark:3.0.3
[root@docker spark]#

验证镜像

[root@docker spark]# docker run -it --rm spark:3.0.3 spark-shell --version
Welcome to____              __/ __/__  ___ _____/ /___\ \/ _ \/ _ `/ __/  '_//___/ .__/\_,_/_/ /_/\_\   version 3.0.3/_/Using Scala version 2.12.10, Java HotSpot(TM) 64-Bit Server VM, 1.8.0_221
Branch HEAD
Compiled by user ubuntu on 2021-06-17T04:52:32Z
Revision 65ac1e75dc468f53fc778cd2ce1ba3f21067aab8
Url https://github.com/apache/spark
Type --help for more information.
[root@docker spark]#