由于apache 官网的 docker image 是单点，要实现集群方式部署。

 

rocketmq 分为 nameserver 和 broker ， 对于之间调用频繁的服务，会增加网络压力， 所以 考虑又网络通信 变成 进程间通信， 那么1个pod 中包含两个container。

 

关于rocketmq 下载编译， https://www.cnblogs.com/fengjian2016/p/10150394.html

拿到 编译后的  apache-rocketmq , 需要修改一些配置， 例如：

1. rm -rf  删除 rocketmq/bin/*.cmd

2 .把runserver.sh  HEAP_OPTS 变成环境变量， 把runbroker.sh  HEAP_OPTS 变成环境变量

  

3. 指定消息存储路径

[root@rocketmq1 conf]#mkdir -p /data/rocketmq/store[root@rocketmq1 conf]#mkdir -p /data/rocketmq/store/commitlog[root@rocketmq1 conf]#mkdir -p /data/rocketmq/logs

4. 指定日志输出路径

/data/rocketmq/conf/ 中的xml文件sed -i 's#${user.home}#/data/rocketmq/logs#g' *.xml

5. 删除多余配置文件 2m-2s 2master 等等， 启动的时候k8s command 直接指向文件 /data/rocketmq/conf/ 目录下新建 broker-a.conf  broker-b.conf 2个文件 6. 配置修改

　　(1) rocketmqHome 修改

　　(2) namesrvAddr 修改，强调  通一个pod 必须写localhost 或者 127.0.0.1 要不然调不通

 

[root@harbor conf]# cat broker-a.conf rocketmqHome=/data/rocketmqnamesrvAddr=localhost:9876;rocketmq2:9876brokerName=broker-abrokerClusterName=DefaultClusterbrokerId=0brokerPermission=6#在发送消息时，自动创建服务器不存在的topic，默认创建的队列数defaultTopicQueueNums=4#是否允许 Broker 自动创建Topic，建议线下开启，线上关闭autoCreateTopicEnable=trueclusterTopicEnable=truebrokerTopicEnable=true#是否允许 Broker 自动创建订阅组，建议线下开启，线上关闭autoCreateSubscriptionGroup=truesendMessageThreadPoolNums=1pullMessageThreadPoolNums=20queryMessageThreadPoolNums=10adminBrokerThreadPoolNums=16clientManageThreadPoolNums=32consumerManageThreadPoolNums=32heartbeatThreadPoolNums=2endTransactionThreadPoolNums=12flushConsumerOffsetInterval=5000flushConsumerOffsetHistoryInterval=60000rejectTransactionMessage=falsefetchNamesrvAddrByAddressServer=falsesendThreadPoolQueueCapacity=10000pullThreadPoolQueueCapacity=100000queryThreadPoolQueueCapacity=20000clientManagerThreadPoolQueueCapacity=1000000consumerManagerThreadPoolQueueCapacity=1000000heartbeatThreadPoolQueueCapacity=50000endTransactionPoolQueueCapacity=100000filterServerNums=0longPollingEnable=trueshortPollingTimeMills=1000notifyConsumerIdsChangedEnable=truehighSpeedMode=falsecommercialEnable=truecommercialTimerCount=1commercialTransCount=1commercialBigCount=1commercialBaseCount=1transferMsgByHeap=truemaxDelayTime=40regionId=DefaultRegionregisterBrokerTimeoutMills=6000slaveReadEnable=falsedisableConsumeIfConsumerReadSlowly=falseconsumerFallbehindThreshold=17179869184brokerFastFailureEnable=truewaitTimeMillsInSendQueue=200waitTimeMillsInPullQueue=5000waitTimeMillsInHeartbeatQueue=31000waitTimeMillsInTransactionQueue=3000startAcceptSendRequestTimeStamp=0traceOn=trueenableCalcFilterBitMap=falseexpectConsumerNumUseFilter=32maxErrorRateOfBloomFilter=20filterDataCleanTimeSpan=86400000filterSupportRetry=falseenablePropertyFilter=falsecompressedRegister=falseforceRegister=trueregisterNameServerPeriod=30000transactionTimeOut=6000transactionCheckMax=15transactionCheckInterval=60000#Broker 对外服务的监听端口listenPort=10911serverWorkerThreads=8serverCallbackExecutorThreads=0serverSelectorThreads=3serverOnewaySemaphoreValue=256serverAsyncSemaphoreValue=64serverChannelMaxIdleTimeSeconds=120serverSocketSndBufSize=131072serverSocketRcvBufSize=131072serverPooledByteBufAllocatorEnable=trueuseEpollNativeSelector=falseclientWorkerThreads=4clientCallbackExecutorThreads=2clientOnewaySemaphoreValue=65535clientAsyncSemaphoreValue=65535connectTimeoutMillis=3000channelNotActiveInterval=60000clientChannelMaxIdleTimeSeconds=120clientSocketSndBufSize=131072clientSocketRcvBufSize=131072clientPooledByteBufAllocatorEnable=falseclientCloseSocketIfTimeout=falseuseTLS=falsestorePathRootDir=/data/rocketmq/storestorePathCommitLog=/root/rocketmq/store/commitlog#commitLog每个文件的大小默认1GmapedFileSizeCommitLog=1073741824#ConsumeQueue每个文件默认存60W条，根据业务情况调整,mapedFileSizeConsumeQueue=3000000enableConsumeQueueExt=falsemappedFileSizeConsumeQueueExt=50331648bitMapLengthConsumeQueueExt=64flushIntervalCommitLog=500commitIntervalCommitLog=200useReentrantLockWhenPutMessage=falseflushCommitLogTimed=falseflushIntervalConsumeQueue=1000cleanResourceInterval=10000deleteCommitLogFilesInterval=100deleteConsumeQueueFilesInterval=100destroyMapedFileIntervalForcibly=120000redeleteHangedFileInterval=120000#删除文件时间点，默认凌晨 2点deleteWhen=02#检测物理文件磁盘空间diskMaxUsedSpaceRatio=75#文件保留时间，默认 72 小时fileReservedTime=128putMsgIndexHightWater=600000#限制的消息大小maxMessageSize=65536checkCRCOnRecover=trueflushCommitLogLeastPages=4commitCommitLogLeastPages=4flushLeastPagesWhenWarmMapedFile=4096flushConsumeQueueLeastPages=2flushCommitLogThoroughInterval=10000commitCommitLogThoroughInterval=200flushConsumeQueueThoroughInterval=60000maxTransferBytesOnMessageInMemory=262144maxTransferCountOnMessageInMemory=32maxTransferBytesOnMessageInDisk=65536maxTransferCountOnMessageInDisk=8accessMessageInMemoryMaxRatio=40messageIndexEnable=truemaxHashSlotNum=5000000maxIndexNum=20000000maxMsgsNumBatch=64messageIndexSafe=falsehaListenPort=10912haSendHeartbeatInterval=5000haHousekeepingInterval=20000haTransferBatchSize=32768haMasterAddress=haSlaveFallbehindMax=268435456brokerRole=ASYNC_MASTERflushDiskType=ASYNC_FLUSHsyncFlushTimeout=5000messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2hflushDelayOffsetInterval=10000cleanFileForciblyEnable=truewarmMapedFileEnable=falseoffsetCheckInSlave=falsedebugLockEnable=falseduplicationEnable=falsediskFallRecorded=trueosPageCacheBusyTimeOutMills=1000defaultQueryMaxNum=32transientStorePoolEnable=falsetransientStorePoolSize=5fastFailIfNoBufferInStorePool=false

 

[root@harbor conf]# cat broker-b.conf rocketmqHome=/data/rocketmqnamesrvAddr=rocketmq1:9876;localhost:9876brokerName=broker-bbrokerClusterName=DefaultClusterbrokerId=0brokerPermission=6#在发送消息时，自动创建服务器不存在的topic，默认创建的队列数defaultTopicQueueNums=4#是否允许 Broker 自动创建Topic，建议线下开启，线上关闭autoCreateTopicEnable=trueclusterTopicEnable=truebrokerTopicEnable=true#是否允许 Broker 自动创建订阅组，建议线下开启，线上关闭autoCreateSubscriptionGroup=truesendMessageThreadPoolNums=1pullMessageThreadPoolNums=20queryMessageThreadPoolNums=10adminBrokerThreadPoolNums=16clientManageThreadPoolNums=32consumerManageThreadPoolNums=32heartbeatThreadPoolNums=2endTransactionThreadPoolNums=12flushConsumerOffsetInterval=5000flushConsumerOffsetHistoryInterval=60000rejectTransactionMessage=falsefetchNamesrvAddrByAddressServer=falsesendThreadPoolQueueCapacity=10000pullThreadPoolQueueCapacity=100000queryThreadPoolQueueCapacity=20000clientManagerThreadPoolQueueCapacity=1000000consumerManagerThreadPoolQueueCapacity=1000000heartbeatThreadPoolQueueCapacity=50000endTransactionPoolQueueCapacity=100000filterServerNums=0longPollingEnable=trueshortPollingTimeMills=1000notifyConsumerIdsChangedEnable=truehighSpeedMode=falsecommercialEnable=truecommercialTimerCount=1commercialTransCount=1commercialBigCount=1commercialBaseCount=1transferMsgByHeap=truemaxDelayTime=40regionId=DefaultRegionregisterBrokerTimeoutMills=6000slaveReadEnable=falsedisableConsumeIfConsumerReadSlowly=falseconsumerFallbehindThreshold=17179869184brokerFastFailureEnable=truewaitTimeMillsInSendQueue=200waitTimeMillsInPullQueue=5000waitTimeMillsInHeartbeatQueue=31000waitTimeMillsInTransactionQueue=3000startAcceptSendRequestTimeStamp=0traceOn=trueenableCalcFilterBitMap=falseexpectConsumerNumUseFilter=32maxErrorRateOfBloomFilter=20filterDataCleanTimeSpan=86400000filterSupportRetry=falseenablePropertyFilter=falsecompressedRegister=falseforceRegister=trueregisterNameServerPeriod=30000transactionTimeOut=6000transactionCheckMax=15transactionCheckInterval=60000#Broker 对外服务的监听端口listenPort=10911serverWorkerThreads=8serverCallbackExecutorThreads=0serverSelectorThreads=3serverOnewaySemaphoreValue=256serverAsyncSemaphoreValue=64serverChannelMaxIdleTimeSeconds=120serverSocketSndBufSize=131072serverSocketRcvBufSize=131072serverPooledByteBufAllocatorEnable=trueuseEpollNativeSelector=falseclientWorkerThreads=4clientCallbackExecutorThreads=2clientOnewaySemaphoreValue=65535clientAsyncSemaphoreValue=65535connectTimeoutMillis=3000channelNotActiveInterval=60000clientChannelMaxIdleTimeSeconds=120clientSocketSndBufSize=131072clientSocketRcvBufSize=131072clientPooledByteBufAllocatorEnable=falseclientCloseSocketIfTimeout=falseuseTLS=falsestorePathRootDir=/data/rocketmq/storestorePathCommitLog=/root/rocketmq/store/commitlog#commitLog每个文件的大小默认1GmapedFileSizeCommitLog=1073741824#ConsumeQueue每个文件默认存60W条，根据业务情况调整,mapedFileSizeConsumeQueue=3000000enableConsumeQueueExt=falsemappedFileSizeConsumeQueueExt=50331648bitMapLengthConsumeQueueExt=64flushIntervalCommitLog=500commitIntervalCommitLog=200useReentrantLockWhenPutMessage=falseflushCommitLogTimed=falseflushIntervalConsumeQueue=1000cleanResourceInterval=10000deleteCommitLogFilesInterval=100deleteConsumeQueueFilesInterval=100destroyMapedFileIntervalForcibly=120000redeleteHangedFileInterval=120000#删除文件时间点，默认凌晨 2点deleteWhen=02#检测物理文件磁盘空间diskMaxUsedSpaceRatio=75#文件保留时间，默认 72 小时fileReservedTime=128putMsgIndexHightWater=600000#限制的消息大小maxMessageSize=65536checkCRCOnRecover=trueflushCommitLogLeastPages=4commitCommitLogLeastPages=4flushLeastPagesWhenWarmMapedFile=4096flushConsumeQueueLeastPages=2flushCommitLogThoroughInterval=10000commitCommitLogThoroughInterval=200flushConsumeQueueThoroughInterval=60000maxTransferBytesOnMessageInMemory=262144maxTransferCountOnMessageInMemory=32maxTransferBytesOnMessageInDisk=65536maxTransferCountOnMessageInDisk=8accessMessageInMemoryMaxRatio=40messageIndexEnable=truemaxHashSlotNum=5000000maxIndexNum=20000000maxMsgsNumBatch=64messageIndexSafe=falsehaListenPort=10912haSendHeartbeatInterval=5000haHousekeepingInterval=20000haTransferBatchSize=32768haMasterAddress=haSlaveFallbehindMax=268435456brokerRole=ASYNC_MASTERflushDiskType=ASYNC_FLUSHsyncFlushTimeout=5000messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2hflushDelayOffsetInterval=10000cleanFileForciblyEnable=truewarmMapedFileEnable=falseoffsetCheckInSlave=falsedebugLockEnable=falseduplicationEnable=falsediskFallRecorded=trueosPageCacheBusyTimeOutMills=1000defaultQueryMaxNum=32transientStorePoolEnable=falsetransientStorePoolSize=5fastFailIfNoBufferInStorePool=false

 

broker Dockerfile

[root@harbor rocketmq-broker]# cat Dockerfile FROM centosMAINTAINER fengjian 
     
      ENV TZ "Asia/Shanghai"ENV TERM xtermENV MALLOC_ARENA_MAX=1ENV JAVA_HOME /data/jdkENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jarENV CATALINA_HOME /data/tomcatENV CATALINA_BASE /data/tomcatENV PATH $PATH:$JAVA_HOME/bin:$CATALINA_HOME/lib:$CATALINA_HOME/binENV ROCKETMQ_HOME /data/rocketmqRUN rm /etc/yum.repos.d/* -rfADD Centos-7.repo /etc/yum.repos.d/ADD epel.repo /etc/yum.repos.d/RUN yum clean allRUN yum -y install net-tools bind-utils sysstatADD localtime  /etc/ADD jdk.tar.gz /dataADD profile /etcADD sysctl.conf /etcADD 20-nproc.conf /etc/security/limits.d/ADD limits.conf /etc/security/ADD rocketmq.tar.gz /data/WORKDIR ${ROCKETMQ_HOME}/binEXPOSE 10909 10911
     

 

 

nameserver  Dockerfile

FROM centosMAINTAINER fengjian 
     
      ENV TZ "Asia/Shanghai"ENV TERM xtermENV MALLOC_ARENA_MAX=1ENV JAVA_HOME /data/jdkENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jarENV CATALINA_HOME /data/tomcatENV CATALINA_BASE /data/tomcatENV PATH $PATH:$JAVA_HOME/bin:$CATALINA_HOME/lib:$CATALINA_HOME/binENV ROCKETMQ_HOME /data/rocketmqRUN rm /etc/yum.repos.d/* -rfADD Centos-7.repo /etc/yum.repos.d/ADD epel.repo /etc/yum.repos.d/RUN yum clean allRUN yum -y install net-tools bind-utils sysstatADD localtime  /etc/ADD jdk.tar.gz /dataADD profile /etcADD sysctl.conf /etcADD 20-nproc.conf /etc/security/limits.d/ADD limits.conf /etc/security/ADD rocketmq.tar.gz /data/WORKDIR ${ROCKETMQ_HOME}/binEXPOSE 9876ENTRYPOINT ["sh","mqnamesrv"]
     

 

cat ceph-rocketmq1-pvc.yaml

kind: PersistentVolumeClaimapiVersion: v1metadata:  name: rocketmq1-brokerstore-pvc  annotations:    volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmqspec:  accessModes:  - ReadWriteOnce  #必须写ReadWriteOnce,否则报错。  resources:    requests:      storage: 20Gi---kind: PersistentVolumeClaimapiVersion: v1metadata:  name: rocketmq1-brokerlog-pvc  annotations:    volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmqspec:  accessModes:  - ReadWriteOnce  #必须写ReadWriteOnce,否则报错。  resources:    requests:      storage: 20Gi---kind: PersistentVolumeClaimapiVersion: v1metadata:  name: rocketmq1-namesrvlog-pvc  annotations:    volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmqspec:  accessModes:  - ReadWriteOnce  #必须写ReadWriteOnce,否则报错。  resources:    requests:      storage: 20Gi

 

rocketmq1 yaml文件

apiVersion: extensions/v1beta1kind: Deploymentmetadata:  name: rocketmq1spec:  replicas: 1  template:    metadata:     labels:       app: rocketmq1    spec:      containers:      - name: namesrv        image: 192.168.200.10/fengjian/rocketmq-nameserver:4.3        imagePullPolicy: Always        ports:          - containerPort: 9876        env:        - name: ROCKETMQ_HEAP_OPTS          value : "-Xms1g -Xmx1g -Xmn512m"        volumeMounts:        - name: rocketmq1namesrvlog          mountPath: /data/rocketmq/logs      - name: broker        image: 192.168.200.10/fengjian/rocketmq-broker:4.3        imagePullPolicy: Always        ports:          - name: brockerserver            containerPort: 10909          - name: brockerslave            containerPort: 10911        command: ["/data/rocketmq/bin/mqbroker"]        args: ["-c", "/data/rocketmq/conf/broker-a.conf"]        env:        - name: ROCKETMQ_HEAP_OPTS          value : "-Xms2g -Xmx2g -Xmn1g"        volumeMounts:          - name: brokerlogs            mountPath: /data/rocketmq/logs          - name: brokerstore            mountPath: /data/rocketmq/store      volumes:      - name: rocketmq1namesrvlog        persistentVolumeClaim:          claimName: rocketmq1-namesrvlog-pvc      - name: brokerlogs        persistentVolumeClaim:          claimName: rocketmq1-brokerlog-pvc      - name: brokerstore        persistentVolumeClaim:          claimName: rocketmq1-brokerstore-pvc---apiVersion: v1kind: Servicemetadata:  name: rocketmq1spec:  ports:  - name: brokerslave    port: 10909    protocol: TCP    targetPort: 10909  - name: brokerserver    port: 10911    protocol: TCP    targetPort: 10911  - name: namesrvport    port: 9876    protocol: TCP    targetPort: 9876  selector:    app: rocketmq1

 

 

还要提前定义pvc 卷

vim  rocketmq2.yaml

apiVersion: extensions/v1beta1kind: Deploymentmetadata:  name: rocketmq1spec:  replicas: 1  template:    metadata:     labels:       app: rocketmq1    spec:      containers:      - name: namesrv        image: 192.168.200.10/fengjian/rocketmq-nameserver:4.3        imagePullPolicy: Always        ports:          - containerPort: 9876        env:        - name: ROCKETMQ_HEAP_OPTS          value : "-Xms1g -Xmx1g -Xmn512m"        volumeMounts:        - name: rocketmq1namesrvlog          mountPath: /data/rocketmq/logs      - name: broker        image: 192.168.200.10/fengjian/rocketmq-broker:4.3        imagePullPolicy: Always        ports:          - name: brockerserver            containerPort: 10909          - name: brockerslave            containerPort: 10911        command: ["/data/rocketmq/bin/mqbroker"]        args: ["-c", "/data/rocketmq/conf/broker-b.conf"]        env:        - name: ROCKETMQ_HEAP_OPTS          value : "-Xms2g -Xmx2g -Xmn1g"        volumeMounts:          - name: brokerlogs            mountPath: /data/rocketmq/logs          - name: brokerstore            mountPath: /data/rocketmq/store      volumes:      - name: rocketmq1namesrvlog        persistentVolumeClaim:          claimName: rocketmq1-namesrvlog-pvc      - name: brokerlogs        persistentVolumeClaim:          claimName: rocketmq1-brokerlog-pvc      - name: brokerstore        persistentVolumeClaim:          claimName: rocketmq1-brokerstore-pvc---apiVersion: v1kind: Servicemetadata:  name: rocketmq1spec:  ports:  - name: brokerslave    port: 10909    protocol: TCP    targetPort: 10909  - name: brokerserver    port: 10911    protocol: TCP    targetPort: 10911  - name: namesrvport    port: 9876    protocol: TCP    targetPort: 9876  selector:    app: rocketmq1