教你在k8s上部署HADOOP-3.2.2(HDFS)的方法教你在k8s上部署HADOOP-3.2.2(HDF

环境+版本
k8s: v1.21.1
hadoop: 3.2.2

dockerfile

FROM openjdk:8-jdk# 如果要通过ssh连接容器内部,添加自己的公钥(非必须)ARG SSH_PUB='ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3nTRJ/aVb67l1xMaN36jmIbabU7Hiv/xpZ8bwLVvNO3Bj7kUzYTp7DIbPcHQg4d6EsPC6j91E8zW6CrV2fo2Ai8tDO/rCq9Se/64F3+8oEIiI6E/OfUZfXD1mPbG7M/kcA3VeQP6wxNPhWBbKRisqgUc6VTKhl+hK6LwRTZgeShxSNcey+HZst52wJxjQkNG+7CAEY5bbmBzAlHCSl4Z0RftYTHR3q8LcEg7YLNZasUogX68kBgRrb+jw1pRMNo7o7RI9xliDAGX+E4C3vVZL0IsccKgr90222axsADoEjC9O+Q6uwKjahemOVaau+9sHIwkelcOcCzW5SuAwkezv 805899926@qq.com'RUN apt-get update; RUN apt-get install -y openssh-server net-tools vim git; RUN sed -i -r 's/^\s*UseDNS\s+\w+/#\0/; s/^\s*PasswordAuthentication\s+\w+/#\0/; s/^\s*ClientAliveInterval\s+\w+/#\0/' /etc/ssh/sshd_config; RUN echo 'UseDNS no \nPermitRootLogin yes \nPasswordAuthentication yes \nClientAliveInterval 30' >> /etc/ssh/sshd_config; RUN cat /etc/ssh/sshd_configRUN su root bash -c 'cd; mkdir .ssh; chmod 700 .ssh; echo ${SSH_PUB} > .ssh/authorized_keys; chmod 644 .ssh/authorized_keys'RUN su root bash -c 'cd; ssh-keygen -t rsa -f ~/.ssh/id_rsa; cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys'# hadoopENV HADOOP_TGZ_URL=https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-3.2.2/hadoop-3.2.2.tar.gzENV HADOOP_HOME=/opt/hadoopENV PATH=$HADOOP_HOME/bin:$PATHRUN set -ex; \mkdir -p $HADOOP_HOME; \wget -nv -O $HADOOP_HOME/src.tgz $HADOOP_TGZ_URL; \tar -xf $HADOOP_HOME/src.tgz --strip-components=1 -C $HADOOP_HOME; \rm $HADOOP_HOME/src.tgz; \chown -R root:root $HADOOP_HOME; \RUN mkdir -p $HADOOP_HOME/hdfs/name/ && mkdir -p $HADOOP_HOME/hdfs/data/# clean trash file or dirRUN rm -rf $HADOOP_HOME/share/doc/; COPY docker-entrypoint.sh /EXPOSE 22 9870 9000ENTRYPOINT ["/docker-entrypoint.sh"]

docker-entrypoint.sh

#!/bin/bashset -eservice ssh starthdfs_dir=$HADOOP_HOME/hdfs/if [ $HADOOP_NODE_TYPE = "datanode" ]; thenecho -e "\033[32m start datanode \033[0m"$HADOOP_HOME/bin/hdfs datanode -regularfiif [ $HADOOP_NODE_TYPE = "namenode" ]; thenif [ -z $(ls -A ${hdfs_dir}) ]; thenecho -e "\033[32m start hdfs namenode format \033[0m"$HADOOP_HOME/bin/hdfs namenode -formatfiecho -e "\033[32m start hdfs namenode \033[0m"$HADOOP_HOME/bin/hdfs namenode

pod template

apiVersion: v1kind: ConfigMapmetadata:name: hadoopnamespace: big-datalabels:app: hadoopdata:hadoop-env.sh: |export HDFS_DATANODE_USER=rootexport HDFS_NAMENODE_USER=rootexport HDFS_SECONDARYNAMENODE_USER=rootexport JAVA_HOME=/usr/local/openjdk-8export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}export HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib/native"core-site.xml: |fs.defaultFShdfs://hadoop-master:9000dfs.namenode.rpc-bind-host0.0.0.0hdfs-site.xml: |dfs.namenode.name.dirfile:///opt/hadoop/hdfs/namedfs.datanode.data.dirfile:///opt/hadoop/hdfs/datadfs.namenode.datanode.registration.ip-hostname-checkfalsedfs.replication1---# namenode svcapiVersion: v1kind: Servicemetadata:name: hadoop-masternamespace: big-dataspec:selector:app: hadoop-namenodetype: NodePortports:- name: rpcport: 9000targetPort: 9000- name: httpport: 9870targetPort: 9870nodePort: 9870# namenode pod---apiVersion: apps/v1kind: Deploymentmetadata:name: hadoop-namenodenamespace: big-dataspec:strategy:type: Recreateselector:matchLabels:app: hadoop-namenodetemplate:metadata:labels:app: hadoop-namenodespec:volumes:- name: hadoop-envconfigMap:name: hadoopitems:- key: hadoop-env.shpath: hadoop-env.sh- name: core-siteconfigMap:name: hadoopitems:- key: core-site.xmlpath: core-site.xml- name: hdfs-siteconfigMap:name: hadoopitems:- key: hdfs-site.xmlpath: hdfs-site.xml- name: hadoop-datapersistentVolumeClaim:claimName: data-hadoop-namenodecontainers:- name: hadoopimage: registry:5000/hadoopimagePullPolicy: Alwaysports:- containerPort: 22- containerPort: 9000- containerPort: 9870volumeMounts:- name: hadoop-envmountPath: /opt/hadoop/etc/hadoop/hadoop-env.shsubPath: hadoop-env.sh- name: core-sitemountPath: /opt/hadoop/etc/hadoop/core-site.xmlsubPath: core-site.xml- name: hdfs-sitemountPath: /opt/hadoop/etc/hadoop/hdfs-site.xmlsubPath: hdfs-site.xml- name: hadoop-datamountPath: /opt/hadoop/hdfs/subPath: hdfs- name: hadoop-datamountPath: /opt/hadoop/logs/subPath: logsenv:- name: HADOOP_NODE_TYPEvalue: namenode---apiVersion: v1kind: PersistentVolumeClaimmetadata:name: data-hadoop-namenodenamespace: big-dataspec:accessModes:- ReadWriteManyresources:requests:storage: 256GistorageClassName: "managed-nfs-storage"# datanode pod---apiVersion: apps/v1kind: StatefulSetmetadata:name: hadoop-datanodenamespace: big-dataspec:replicas: 2selector:matchLabels:app: hadoop-datanodeserviceName: hadoop-datanodetemplate:metadata:labels:app: hadoop-datanodespec:volumes:- name: hadoop-envconfigMap:name: hadoopitems:- key: hadoop-env.shpath: hadoop-env.sh- name: core-siteconfigMap:name: hadoopitems:- key: core-site.xmlpath: core-site.xml- name: hdfs-siteconfigMap:name: hadoopitems:- key: hdfs-site.xmlpath: hdfs-site.xmlcontainers:- name: hadoopimage: registry:5000/hadoopimagePullPolicy: Alwaysports:- containerPort: 22- containerPort: 9000- containerPort: 9870volumeMounts:- name: hadoop-envmountPath: /opt/hadoop/etc/hadoop/hadoop-env.shsubPath: hadoop-env.sh- name: core-sitemountPath: /opt/hadoop/etc/hadoop/core-site.xmlsubPath: core-site.xml- name: hdfs-sitemountPath: /opt/hadoop/etc/hadoop/hdfs-site.xmlsubPath: hdfs-site.xml- name: datamountPath: /opt/hadoop/hdfs/subPath: hdfs- name: datamountPath: /opt/hadoop/logs/subPath: logsenv:- name: HADOOP_NODE_TYPEvalue: datanodevolumeClaimTemplates:- metadata:name: datanamespace: big-dataspec:accessModes:- ReadWriteManyresources:requests:storage: 256GistorageClassName: "managed-nfs-storage"