一、針對于kafka版本
<dependency><groupId>org.apache.kafka</groupId><artifactId>kafka-clients</artifactId><version>0.8.2.2</version></dependency><dependency><groupId>org.apache.kafka</groupId><artifactId>kafka_2.11</artifactId><version>0.8.2.2</version> </dependency>
二、操作topic、consumer相關方法
import kafka.admin.AdminUtils; import kafka.admin.TopicCommand; import kafka.api.TopicMetadata; import kafka.tools.ConsumerOffsetChecker; import kafka.utils.ZKStringSerializer$; import kafka.utils.ZkUtils; import org.I0Itec.zkclient.ZkClient; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.ExecuteWatchdog; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.tools.ant.taskdefs.Execute;public static class KafkaUtils {private static Logger LOGGER = LoggerFactory.getLogger(KafkaUtils.class);private static AutoZkClient zkClient;public static AutoZkClient getZkClient() {return zkClient;}public static void setZkClient(AutoZkClient zkClient) {KafkaUtils.zkClient = zkClient;}public static boolean topicExists(String topic) {Assert.notNull(zkClient, "zkclient is null");return AdminUtils.topicExists(zkClient, topic);}public static void topicChangeConfig(String topic, Properties properties) {Assert.notNull(zkClient, "zkclient is null");AdminUtils.changeTopicConfig(zkClient, topic, properties);}public static void topicAlterPartitions(String topic, int partitions) {Assert.notNull(zkClient, "zkclient is null");TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient);int curPartitions = topicMetadata.partitionsMetadata().size();if (curPartitions == partitions) {return;}if (curPartitions > partitions) {LOGGER.info(String.format("curPartitions=%d，不能修改partitions=%d，請確保大與當前分區數", curPartitions, partitions));return;}String[] args = {"--zookeeper", zkClient.zkServers,"--partitions", String.valueOf(partitions),"--alter","--topic", topic};TopicCommand.TopicCommandOptions alterOpt = new TopicCommand.TopicCommandOptions(args);alterOpt.checkArgs();TopicCommand.alterTopic(zkClient, alterOpt);}public static void topicDescribe(String topic) {Assert.notNull(zkClient, "zkclient is null");String[] args = {"--zookeeper", zkClient.zkServers,"--describe","--topic", topic};TopicCommand.TopicCommandOptions describeOpt = new TopicCommand.TopicCommandOptions(args);describeOpt.checkArgs();TopicCommand.describeTopic(zkClient, describeOpt);}public static void topicOverrideConfig(String topic, Properties properties) {Assert.notNull(zkClient, "zkclient is null");Properties oldProperties = KafkaUtils.topicConfig(topic);oldProperties.putAll(properties);AdminUtils.changeTopicConfig(zkClient, topic, oldProperties);}public static void topicCreate(TopicConfig topicConfig) {Assert.notNull(zkClient, "zkclient is null");int brokerSize = ZkUtils.getSortedBrokerList(zkClient).size();if (topicConfig.getReplicationFactor() > brokerSize) {topicConfig.setReplicationFactor(brokerSize);LOGGER.info(String.format("broker-size=%d < replication-factor=%d, 所以設置replication-factor大小為broker-size大小", brokerSize, topicConfig.getReplicationFactor()));}AdminUtils.createTopic(zkClient, topicConfig.getName(), topicConfig.getPartitions(), topicConfig.getReplicationFactor(), topicConfig.getProperties());}public static void topicDelete(String topic) {Assert.notNull(zkClient, "zkclient is null");AdminUtils.deleteTopic(zkClient, topic);}public static List<String> topicsList() {Assert.notNull(zkClient, "zkclient is null");return seqAsJavaList(ZkUtils.getAllTopics(zkClient));}public static Properties topicConfig(String topic) {Assert.notNull(zkClient, "zkclient is null");return AdminUtils.fetchTopicConfig(zkClient, topic);}public static Map<String, Properties> topicsConfig() {Assert.notNull(zkClient, "zkclient is null");return mapAsJavaMap(AdminUtils.fetchAllTopicConfigs(zkClient));}public static void consumerDetail(String topic, String group){String[] args = {"--zookeeper", zkClient.getZkServers(),"--group", group,"--topic", topic};ConsumerOffsetChecker.main(args);}public static Map<String, List<String>> getConsumersPerTopic(String group) {return mapAsJavaMap(ZkUtils.getConsumersPerTopic(zkClient, group, false)).entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey(), entry -> JavaConversions.seqAsJavaList(entry.getValue()).stream().map(consumerThreadId -> consumerThreadId.consumer()).collect(Collectors.toList())));}public static List<String> getConsumersInGroup(String group){return JavaConversions.seqAsJavaList(ZkUtils.getConsumersInGroup(zkClient, group));}public static String executeCommond(String commond) {LOGGER.info("begin to execute commond: " + commond);File tmpFileDir = Files.createTempDir();String tmpFileName = UUID.randomUUID().toString() + ".txt";String fileSavePath = tmpFileDir.getAbsolutePath() + tmpFileName;CommandLine oCmdLine = CommandLine.parse(commond + " > " + fileSavePath);DefaultExecutor executor = new DefaultExecutor();ExecuteWatchdog watchdog = new ExecuteWatchdog(20000);executor.setWatchdog(watchdog);int[] exitValues = {0, 1};executor.setExitValues(exitValues);try {if (Execute.isFailure(executor.execute(oCmdLine))) {watchdog.killedProcess();LOGGER.error("遠程命令執行失敗... commond=" + commond);} else {try (Stream<String> lines = java.nio.file.Files.lines(new File(fileSavePath).toPath(), Charset.defaultCharset())) {List<String> fileLines = lines.collect(toCollection(LinkedList::new));StringBuilder result = new StringBuilder();fileLines.forEach(line -> result.append(line).append(System.lineSeparator()));return result.toString();} finally {FileUtils.deleteQuietly(tmpFileDir);}}} catch (Exception e) {LOGGER.error("execute command error happened... commond=" + commond, e);}return StringUtils.EMPTY;} }
三、控制層展示
import com.alibaba.fastjson.JSON; import com.cmos.common.annotation.CompatibleOutput; import com.cmos.core.logger.Logger; import com.cmos.core.logger.LoggerFactory; import com.cmos.wmhopenapi.web.config.KafkaMessageConfig; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; import scala.Console;import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.List;import static com.cmos.wmhopenapi.web.config.KafkaMessageConfig.KafkaUtils;/*** @author hujunzheng* @create 2018-07-16 10:20**/ @RestController @RequestMapping("/message/state") @CompatibleOutput public class MessageCenterStateController {private static Logger LOGGER = LoggerFactory.getLogger(MessageCenterStateController.class);@Autowiredprivate KafkaMessageConfig.NoAckConsumer noAckConsumer;@Autowiredprivate KafkaMessageConfig.AckConsumer ackConsumer;/*** 獲取topic描述** @param topic**/@GetMapping("/topic-describe")public String topicDescribe(@RequestParam String topic) {try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {//scala 輸出流重定向Console.setOut(new PrintStream(bos));KafkaUtils.topicDescribe(topic);String result = bos.toString();LOGGER.info(result);return String.format("%s%s%s", "<pre>", result, "</pre>");} catch (Exception e) {LOGGER.error("獲取topic描述異常", e);}return StringUtils.EMPTY;}/*** 獲取全部topic**/@GetMapping(value = "/topics-all", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)public String topicAll() {String result = JSON.toJSONString(KafkaUtils.topicsList());LOGGER.info(result);return result;}/*** 獲取topic配置** @param topic**/@GetMapping(value = "/topic-config", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)public String topicConfig(@RequestParam String topic) {String result = JSON.toJSONString(KafkaUtils.topicConfig(topic));LOGGER.info(result);return result;}/*** 獲取所有topic的配置**/@GetMapping(value = "/topics-configs", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)public String topicsConfigs() {String result = JSON.toJSONString(KafkaUtils.topicsConfig());LOGGER.info(result);return result;}/*** 展示在某個分組中的consumer** @param group**/@GetMapping(value = "/consumers-in-group", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)public String consumersInGroup(@RequestParam String group) {String result = JSON.toJSONString(KafkaUtils.getConsumersInGroup(group));LOGGER.info(result);return result;}/*** 展示在某個分組中的consumer，按照topic劃分** @param group**/@GetMapping(value = "/consumers-per-topic", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)public String consumersPerTopic(@RequestParam String group) {String result = JSON.toJSONString(KafkaUtils.getConsumersPerTopic(group));LOGGER.info(result);return result;}/*** 展示消費者消費詳情** @param topic* @param group**/@GetMapping("/consumer-detail")public String consumerDetail(@RequestParam String topic, @RequestParam String group) {try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {//scala 輸出流重定向Console.setOut(new PrintStream(bos));KafkaUtils.consumerDetail(topic, group);String result = bos.toString();LOGGER.info(result);return String.format("%s%s%s", "<pre>", result, "</pre>");} catch (Exception e) {LOGGER.error("獲取消費詳情", e);}return StringUtils.EMPTY;}/*** 消費消息并展示消息** @param topic* @param group* @param size 消費消息數量* @param ack 消費的消息是否需要進行ack操作**/@GetMapping("/consumer-consume")public String consumerConsume(@RequestParam String topic, @RequestParam(required = false, defaultValue = "default") String group, @RequestParam(required = false, defaultValue = "1") int size, @RequestParam(required = false, defaultValue = "false") boolean ack) {List<String> messages;if (ack) {messages = ackConsumer.consume(topic, group, size);} else {messages = noAckConsumer.consume(topic, group, size);}return JSON.toJSONString(messages);}/*** 運行kafka相關命令** @param sshRemote 連接遠程主機命令（ssh user@host）* @param sshCommond kafka相關命令 (kafka-consumer.sh ...)**/@PostMapping("/commond-execute")public String commondExecute(@RequestParam(required = false) String sshRemote, @RequestParam String sshCommond) {String commond = sshCommond + StringUtils.EMPTY;if (StringUtils.isNotBlank(sshRemote)) {commond = String.format("%s \"%s\"", sshRemote, commond);}String result = KafkaUtils.executeCommond(commond);return String.format("%s%s%s", "<pre>", result, "</pre>");} }
四、消費配置
消費邏輯
package com.mochasoft.latte.data.kafka.consumer;import java.util.Properties; import kafka.consumer.ConsumerConfig; import org.apache.commons.lang3.StringUtils;public class KafkaConsumerConfig {private String zkConnect;private String zkSessionTimeoutMs;private String zkSyncTimeMs;private String autoCommitIntervalMs;private String groupId = "default";static enum OffSet{smallest, largest;private OffSet() {}}private OffSet offset = OffSet.largest;private Properties properties;public KafkaConsumerConfig(){this.properties = new Properties();}public KafkaConsumerConfig(String zkConnect, String zkSessionTimeoutMs, String zkSyncTimeMs, String autoCommitIntervalMs){this.zkConnect = zkConnect;this.zkSessionTimeoutMs = zkSessionTimeoutMs;this.zkSyncTimeMs = zkSyncTimeMs;this.autoCommitIntervalMs = autoCommitIntervalMs;this.properties = new Properties();}public String getZkConnect(){return this.zkConnect;}public void setZkConnect(String zkConnect){this.zkConnect = zkConnect;}public String getZkSessionTimeoutMs(){return this.zkSessionTimeoutMs;}public void setZkSessionTimeoutMs(String zkSessionTimeoutMs){this.zkSessionTimeoutMs = zkSessionTimeoutMs;}public String getZkSyncTimeMs(){return this.zkSyncTimeMs;}public void setZkSyncTimeMs(String zkSyncTimeMs){this.zkSyncTimeMs = zkSyncTimeMs;}public String getAutoCommitIntervalMs(){return this.autoCommitIntervalMs;}public void setAutoCommitIntervalMs(String autoCommitIntervalMs){this.autoCommitIntervalMs = autoCommitIntervalMs;}public String getGroupId(){return this.groupId;}public void setGroupId(String groupId){if (StringUtils.isNotBlank(groupId)) {this.groupId = groupId;}}public OffSet getOffset(){return this.offset;}public void setOffset(OffSet offset){this.offset = offset;}public ConsumerConfig getConsumerConfig(){return new ConsumerConfig(getProperties());}public Properties getProperties(){if (StringUtils.isBlank(this.zkConnect)) {throw new IllegalArgumentException("Blank zkConnect");}if (StringUtils.isNotBlank(this.zkSessionTimeoutMs)) {this.properties.put("zookeeper.session.timeout.ms", this.zkSessionTimeoutMs);}if (StringUtils.isNotBlank(this.zkSyncTimeMs)) {this.properties.put("zookeeper.sync.time.ms", this.zkSyncTimeMs);}if (StringUtils.isNotBlank(this.autoCommitIntervalMs)) {this.properties.put("auto.commit.interval.ms", this.autoCommitIntervalMs);}if (StringUtils.isNotBlank(this.offset.name())) {this.properties.put("auto.offset.reset", this.offset.name());}this.properties.put("group.id", getGroupId());this.properties.put("zookeeper.connect", this.zkConnect);return this.properties;} }
public static final class NoAckConsumer extends TheConsumer {public NoAckConsumer(KafkaConsumerConfig kafkaConsumerConfig) {super(kafkaConsumerConfig, false);this.consumerConfigProperties.setProperty("auto.commit.enable", "false");} }public static final class AckConsumer extends TheConsumer {public AckConsumer(KafkaConsumerConfig kafkaConsumerConfig) {super(kafkaConsumerConfig, true);this.consumerConfigProperties.setProperty("auto.commit.enable", "true");} }public static class TheConsumer {protected Properties consumerConfigProperties;private boolean ack;private StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());private StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());public TheConsumer(KafkaConsumerConfig kafkaConsumerConfig, boolean ack) {this.ack = ack;this.consumerConfigProperties = new Properties();this.consumerConfigProperties.putAll(kafkaConsumerConfig.getProperties());}/*** @param topic 主題* @param group 分組* @param size 消費數量**/public List<String> consume(String topic, String group, int size) {if (StringUtils.isNotBlank(group)) {this.consumerConfigProperties.setProperty("group.id", group);}ConsumerConnector consumerConnector = null;try {consumerConnector = Consumer.createJavaConsumerConnector(new ConsumerConfig(this.consumerConfigProperties));Map<String, Integer> topics = new HashMap<>(1);topics.put(topic, 1);Map<String, List<KafkaStream<String, String>>> streams = consumerConnector.createMessageStreams(topics, keyDecoder, valueDecoder);if (!(CollectionUtils.isEmpty(streams) || CollectionUtils.isEmpty(streams.get(topic)))) {List<String> messages = new ArrayList<>();KafkaStream<String, String> messageStream = streams.get(topic).get(0);for (ConsumerIterator<String, String> it = messageStream.iterator(); it.hasNext(); ) {MessageAndMetadata<String, String> messageAndMetadata = it.next();messages.add(messageAndMetadata.message());if (this.ack) {consumerConnector.commitOffsets();}if (size <= messages.size()) {break;}}return messages;}} catch (Exception e) {LOGGER.error(String.format("%s ack consume has errors. topic=%s, group=%s, size=%d.", this.ack ? "" : "no", topic, group, size), e);} finally {if (consumerConnector != null) {consumerConnector.shutdown();}}return Collections.EMPTY_LIST;} }
消費測試
public class KafkaTest extends BaseUnitTest {private static Logger LOGGER = LoggerFactory.getLogger(KafkaTest.class);@Value("${kafka.connection.zkconnect}")private String zkConnect;private static final AutoZkClient zkClient = new AutoZkClient("127.0.0.1:2181");private static final String TEST_TOPIC = "message-center-biz-expiration-reminder-topic";private static final String TEST_GROUP = "hjz-group";@Autowiredprivate NoAckConsumer noAckConsumer;@Autowiredprivate AckConsumer ackConsumer;@Autowiredprivate KafkaProducer kafkaProducer;private CountDownLatch finishCountDownLatch = new CountDownLatch(20);@Testpublic void testNoAckConsume() throws InterruptedException {class ConsumeRun implements Callable<List<String>> {private TheConsumer consumer;private CountDownLatch countDownLatch;public ConsumeRun(TheConsumer consumer, CountDownLatch countDownLatch) {this.consumer = consumer;this.countDownLatch = countDownLatch;}@Overridepublic List<String> call() {try {this.countDownLatch.await();} catch (InterruptedException e) {e.printStackTrace();}List<String> messages = consumer.consume(TEST_TOPIC, TEST_GROUP, 8);finishCountDownLatch.countDown();return messages;}}ExecutorService executorService = Executors.newFixedThreadPool(20);CountDownLatch countDownLatch = new CountDownLatch(1);List<Future<List<String>>> noAckConsumeFutures = new ArrayList<>(), ackConsumeFutures = new ArrayList<>();for (int i = 0; i < 10; ++i) {ConsumeRun consumeRun = new ConsumeRun(this.noAckConsumer, countDownLatch);noAckConsumeFutures.add(executorService.submit(consumeRun));}for (int i = 0; i < 10; ++i) {ConsumeRun consumeRun = new ConsumeRun(this.ackConsumer, countDownLatch);ackConsumeFutures.add(executorService.submit(consumeRun));}countDownLatch.countDown();finishCountDownLatch.await();System.out.println("no ack consumers response....");noAckConsumeFutures.forEach(future -> {try {System.out.println(future.get());} catch (Exception e){}});System.out.println("\n\nack consumers response....");ackConsumeFutures.forEach(future -> {try {System.out.println(future.get());} catch (Exception e) {e.printStackTrace();}});}@Testpublic void testProduce() {for (int i = 0; i < 100; ++i) {kafkaProducer.send(TEST_TOPIC, String.valueOf(i), "message " + i);}KafkaUtils.consumerDetail(TEST_TOPIC, TEST_GROUP);}public static void createTopic() {MessageCenterConstants.TopicConfig topicConfig = new MessageCenterConstants.TopicConfig();topicConfig.setName("kafka-test");KafkaMessageConfig.KafkaUtils.topicCreate(topicConfig);}public static void testKafka() {createTopic();System.out.println(KafkaUtils.topicsList());Properties properties = new Properties();properties.put("min.cleanable.dirty.ratio", "0.3");KafkaMessageConfig.KafkaUtils.topicChangeConfig(TEST_TOPIC, properties);System.out.println(KafkaMessageConfig.KafkaUtils.topicConfig(TEST_TOPIC));KafkaUtils.topicAlterPartitions(TEST_TOPIC, 7);KafkaMessageConfig.KafkaUtils.topicDescribe(TEST_TOPIC);kafka.utils.ZkUtils.getSortedBrokerList(zkClient);}public static void testTopicDescribe() {KafkaUtils.setZkClient(zkClient);new MessageCenterStateController().topicDescribe("message-center-recharge-transaction-push-topic");}public static void testConsumerDescribe() {KafkaUtils.setZkClient(zkClient);String[] args = {"--zookeeper", zkClient.getZkServers(),"--group", "","--topic", "message-center-recharge-transaction-push-topic"};ConsumerOffsetChecker.main(args);}public static void testConsumerList() {KafkaUtils.setZkClient(zkClient);String[] args = {"--broker-list", zkClient.getZkServers(),"--topic", "message-center-recharge-transaction-push-topic","--list"};SimpleConsumerShell.main(args);}public static void main(String[] args) {testConsumerList();} }
測試no ack 以及 ack的消費結果
no ack consumers response.... [message 8, message 14, message 23, message 32, message 41, message 50, message 8, message 14] [message 14, message 23, message 32, message 41, message 50, message 12, message 21, message 30] [message 17, message 26, message 35, message 44, message 53, message 62, message 71, message 80] [message 19, message 28, message 37, message 46, message 55, message 64, message 73, message 82] [message 89, message 98, message 89, message 98, message 19, message 28, message 37, message 46] [message 0, message 39, message 48, message 57, message 66, message 75, message 84, message 93] [message 1, message 49, message 58, message 67, message 76, message 85, message 94, message 77] [message 8, message 14, message 23, message 32, message 41, message 50, message 89, message 98] [message 17, message 26, message 35, message 44, message 53, message 62, message 71, message 80] [message 2, message 59, message 68, message 77, message 86, message 95, message 0, message 39]ack consumers response.... [message 7, message 13, message 22, message 31, message 40, message 5, message 11, message 20] [message 17, message 26, message 35, message 44, message 53, message 62, message 71, message 80] [message 77, message 86, message 95, message 67, message 76, message 85, message 94, message 0] [message 9, message 15, message 24, message 33, message 42, message 51, message 60, message 6] [message 4, message 10, message 79, message 88, message 97, message 2, message 59, message 68] [message 29, message 38, message 47, message 56, message 65, message 74, message 83, message 92] [message 16, message 25, message 34, message 43, message 52, message 61, message 70, message 8] [message 18, message 27, message 36, message 45, message 54, message 63, message 72, message 81] [message 3, message 69, message 78, message 87, message 96, message 1, message 49, message 58] [message 14, message 23, message 32, message 41, message 50, message 89, message 98, message 12]
消費測試結果分析：no ack的consumer可以實現消息的窺探。
?
五、效果圖
獲取topic詳情
獲取所有的topic信息
? 獲取消費詳情
? 
? 窺探消息
?
實現方案)
實現方案續)
