HBase分片技術實現

概述

HBase是基于Hadoop的分布式、可擴展的NoSQL數據庫，采用列族存儲模型。HBase的分片機制通過Region自動分割和負載均衡實現水平擴展，支持PB級數據存儲和高并發訪問。

HBase架構

核心組件

HMaster: 集群管理節點，負責Region分配和負載均衡
RegionServer: 數據存儲節點，管理多個Region
Region: 數據分片單元，按行鍵范圍分割
ZooKeeper: 協調服務，維護集群狀態
HDFS: 底層存儲系統

分片原理

表 (Table)
├── Region 1 [startKey, endKey1)
├── Region 2 [endKey1, endKey2)
├── Region 3 [endKey2, endKey3)
└── Region N [endKeyN-1, endKey)

環境搭建

Docker Compose配置

version:?'3.8'
services:zookeeper:image:?zookeeper:3.7container_name:?hbase-zookeeperports:-?"2181:2181"environment:ZOO_MY_ID:?1ZOO_SERVERS:?server.1=0.0.0.0:2888:3888;2181volumes:-?zk_data:/data-?zk_logs:/dataloghbase-master:image:?harisekhon/hbase:2.4container_name:?hbase-masterhostname:?hbase-masterports:-?"16010:16010"??# HBase Master Web UI-?"16000:16000"??# HBase Master RPCenvironment:HBASE_CONF_hbase_rootdir:?hdfs://namenode:9000/hbaseHBASE_CONF_hbase_cluster_distributed:?'true'HBASE_CONF_hbase_zookeeper_quorum:?zookeeper:2181HBASE_CONF_hbase_master:?hbase-master:16000HBASE_CONF_hbase_master_hostname:?hbase-masterHBASE_CONF_hbase_master_port:?16000HBASE_CONF_hbase_master_info_port:?16010HBASE_CONF_hbase_regionserver_port:?16020HBASE_CONF_hbase_regionserver_info_port:?16030depends_on:-?zookeeper-?namenodevolumes:-?hbase_data:/opt/hbase/datahbase-regionserver1:image:?harisekhon/hbase:2.4container_name:?hbase-regionserver1hostname:?hbase-regionserver1ports:-?"16030:16030"??# RegionServer Web UI-?"16020:16020"??# RegionServer RPCenvironment:HBASE_CONF_hbase_rootdir:?hdfs://namenode:9000/hbaseHBASE_CONF_hbase_cluster_distributed:?'true'HBASE_CONF_hbase_zookeeper_quorum:?zookeeper:2181HBASE_CONF_hbase_master:?hbase-master:16000HBASE_CONF_hbase_regionserver_hostname:?hbase-regionserver1HBASE_CONF_hbase_regionserver_port:?16020HBASE_CONF_hbase_regionserver_info_port:?16030depends_on:-?hbase-mastervolumes:-?hbase_rs1_data:/opt/hbase/datahbase-regionserver2:image:?harisekhon/hbase:2.4container_name:?hbase-regionserver2hostname:?hbase-regionserver2ports:-?"16031:16030"??# RegionServer Web UI-?"16021:16020"??# RegionServer RPCenvironment:HBASE_CONF_hbase_rootdir:?hdfs://namenode:9000/hbaseHBASE_CONF_hbase_cluster_distributed:?'true'HBASE_CONF_hbase_zookeeper_quorum:?zookeeper:2181HBASE_CONF_hbase_master:?hbase-master:16000HBASE_CONF_hbase_regionserver_hostname:?hbase-regionserver2HBASE_CONF_hbase_regionserver_port:?16020HBASE_CONF_hbase_regionserver_info_port:?16030depends_on:-?hbase-mastervolumes:-?hbase_rs2_data:/opt/hbase/datanamenode:image:?apache/hadoop:3.3.4container_name:?hadoop-namenodehostname:?namenodeports:-?"9870:9870"??# Namenode Web UI-?"9000:9000"??# Namenode RPCenvironment:CLUSTER_NAME:?hadoop-clustercommand:?["/opt/hadoop/bin/hdfs",?"namenode"]volumes:-?namenode_data:/opt/hadoop/datadatanode:image:?apache/hadoop:3.3.4container_name:?hadoop-datanodehostname:?datanodeports:-?"9864:9864"??# Datanode Web UIenvironment:CLUSTER_NAME:?hadoop-clustercommand:?["/opt/hadoop/bin/hdfs",?"datanode"]depends_on:-?namenodevolumes:-?datanode_data:/opt/hadoop/datavolumes:zk_data:zk_logs:hbase_data:hbase_rs1_data:hbase_rs2_data:namenode_data:datanode_data:

初始化腳本

#!/bin/bash
# init-hbase.shecho?"啟動HBase集群..."
docker-compose up -decho?"等待服務啟動..."
sleep 60echo?"創建測試表..."
docker?exec?-it hbase-master hbase shell <<?'EOF'
create?'user_table',?'info',?'stats'
create?'order_table',?'detail',?'payment'
create?'log_table',?'content'
list
EOFecho?"HBase集群初始化完成"
echo?"HBase Master Web UI: http://localhost:16010"
echo?"RegionServer1 Web UI: http://localhost:16030"
echo?"RegionServer2 Web UI: http://localhost:16031"

Java應用集成

Maven依賴

<dependencies><dependency><groupId>org.apache.hbase</groupId><artifactId>hbase-client</artifactId><version>2.4.17</version></dependency><dependency><groupId>org.apache.hbase</groupId><artifactId>hbase-common</artifactId><version>2.4.17</version></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter</artifactId></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-web</artifactId></dependency>
</dependencies>

Spring Boot配置

@Configuration
@EnableConfigurationProperties(HBaseProperties.class)
public?class?HBaseConfig?{@Autowiredprivate?HBaseProperties hbaseProperties;@Beanpublic?Connection?hbaseConnection()?throws?IOException?{org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create();// 設置ZooKeeper連接config.set("hbase.zookeeper.quorum", hbaseProperties.getZookeeperQuorum());config.set("hbase.zookeeper.property.clientPort", hbaseProperties.getZookeeperPort());// 設置HBase連接參數config.set("hbase.client.retries.number",?"3");config.set("hbase.client.pause",?"1000");config.set("hbase.rpc.timeout",?"60000");config.set("hbase.client.operation.timeout",?"120000");config.set("hbase.client.scanner.timeout.period",?"120000");return?ConnectionFactory.createConnection(config);}@Beanpublic?Admin?hbaseAdmin(Connection connection)?throws?IOException?{return?connection.getAdmin();}
}@ConfigurationProperties(prefix =?"hbase")
@Data
public?class?HBaseProperties?{private?String zookeeperQuorum =?"localhost";private?String zookeeperPort =?"2181";private?int?maxConnections =?100;private?int?coreConnections =?10;
}

HBase操作服務

@Service
@Slf4j
public?class?HBaseService?{@Autowiredprivate?Connection connection;@Autowiredprivate?Admin admin;/*** 創建表*/public?void?createTable(String tableName, String... columnFamilies)?{try?{TableName table = TableName.valueOf(tableName);if?(admin.tableExists(table)) {log.warn("表已存在: {}", tableName);return;}TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(table);// 添加列族for?(String cf : columnFamilies) {ColumnFamilyDescriptor cfDesc = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(cf)).setMaxVersions(3).setTimeToLive(86400?*?30)?// 30天TTL.setCompressionType(Compression.Algorithm.SNAPPY).build();builder.setColumnFamily(cfDesc);}// 預分區byte[][] splitKeys = generateSplitKeys(tableName);admin.createTable(builder.build(), splitKeys);log.info("創建表成功: {}", tableName);}?catch?(IOException e) {log.error("創建表失敗: {}", tableName, e);throw?new?RuntimeException(e);}}/*** 生成預分區鍵*/private?byte[][] generateSplitKeys(String tableName) {List<byte[]> splitKeys =?new?ArrayList<>();if?(tableName.contains("user")) {// 用戶表按用戶ID前綴分區for?(int?i =?1; i <?16; i++) {splitKeys.add(Bytes.toBytes(String.format("%02x", i)));}}?else?if?(tableName.contains("order")) {// 訂單表按時間分區LocalDate start = LocalDate.now().minusMonths(12);for?(int?i =?0; i <?12; i++) {String partition = start.plusMonths(i).format(DateTimeFormatter.ofPattern("yyyyMM"));splitKeys.add(Bytes.toBytes(partition));}}?else?{// 默認按哈希分區for?(int?i =?1; i <?10; i++) {splitKeys.add(Bytes.toBytes(String.valueOf(i)));}}return?splitKeys.toArray(new?byte[0][]);}/*** 插入數據*/public?void?put(String tableName, String rowKey, String columnFamily,?String column, String value)?{try?(Table table = connection.getTable(TableName.valueOf(tableName))) {Put put =?new?Put(Bytes.toBytes(rowKey));put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column),?Bytes.toBytes(value));table.put(put);}?catch?(IOException e) {log.error("插入數據失敗", e);throw?new?RuntimeException(e);}}/*** 批量插入*/public?void?batchPut(String tableName, List<Put> puts)?{try?(Table table = connection.getTable(TableName.valueOf(tableName))) {table.put(puts);log.info("批量插入數據: {} 條", puts.size());}?catch?(IOException e) {log.error("批量插入失敗", e);throw?new?RuntimeException(e);}}/*** 獲取數據*/public?Result?get(String tableName, String rowKey)?{try?(Table table = connection.getTable(TableName.valueOf(tableName))) {Get get =?new?Get(Bytes.toBytes(rowKey));return?table.get(get);}?catch?(IOException e) {log.error("獲取數據失敗", e);throw?new?RuntimeException(e);}}/*** 掃描數據*/public?List<Result>?scan(String tableName, String startRow, String stopRow)?{List<Result> results =?new?ArrayList<>();try?(Table table = connection.getTable(TableName.valueOf(tableName))) {Scan scan =?new?Scan();if?(startRow !=?null) {scan.withStartRow(Bytes.toBytes(startRow));}if?(stopRow !=?null) {scan.withStopRow(Bytes.toBytes(stopRow));}try?(ResultScanner scanner = table.getScanner(scan)) {for?(Result result : scanner) {results.add(result);}}}?catch?(IOException e) {log.error("掃描數據失敗", e);throw?new?RuntimeException(e);}return?results;}/*** 刪除數據*/public?void?delete(String tableName, String rowKey)?{try?(Table table = connection.getTable(TableName.valueOf(tableName))) {Delete delete =?new?Delete(Bytes.toBytes(rowKey));table.delete(delete);}?catch?(IOException e) {log.error("刪除數據失敗", e);throw?new?RuntimeException(e);}}
}

分片管理服務

@Service
@Slf4j
public?class?HBaseShardingService?{@Autowiredprivate?Connection connection;@Autowiredprivate?Admin admin;/*** 獲取表的Region信息*/public?List<RegionInfo>?getTableRegions(String tableName)?{try?{TableName table = TableName.valueOf(tableName);return?admin.getRegions(table);}?catch?(IOException e) {log.error("獲取Region信息失敗", e);throw?new?RuntimeException(e);}}/*** 手動分割Region*/public?void?splitRegion(String tableName, String splitKey)?{try?{TableName table = TableName.valueOf(tableName);admin.split(table, Bytes.toBytes(splitKey));log.info("手動分割Region: {} at {}", tableName, splitKey);}?catch?(IOException e) {log.error("分割Region失敗", e);throw?new?RuntimeException(e);}}/*** 合并Region*/public?void?mergeRegions(String tableName, String region1, String region2)?{try?{admin.mergeRegionsAsync(Bytes.toBytes(region1),Bytes.toBytes(region2),false);log.info("合并Region: {} + {}", region1, region2);}?catch?(IOException e) {log.error("合并Region失敗", e);throw?new?RuntimeException(e);}}/*** 移動Region*/public?void?moveRegion(String regionName, String targetServer)?{try?{admin.move(Bytes.toBytes(regionName), ServerName.valueOf(targetServer));log.info("移動Region: {} to {}", regionName, targetServer);}?catch?(IOException e) {log.error("移動Region失敗", e);throw?new?RuntimeException(e);}}/*** 負載均衡*/public?void?balanceCluster()?{try?{boolean?result = admin.balance();log.info("集群負載均衡: {}", result ??"成功"?:?"無需均衡");}?catch?(IOException e) {log.error("負載均衡失敗", e);throw?new?RuntimeException(e);}}/*** 獲取集群狀態*/public?ClusterMetrics?getClusterStatus()?{try?{return?admin.getClusterMetrics();}?catch?(IOException e) {log.error("獲取集群狀態失敗", e);throw?new?RuntimeException(e);}}/*** 監控Region分布*/@Scheduled(fixedRate =?300000)?// 5分鐘public?void?monitorRegionDistribution()?{try?{ClusterMetrics metrics = getClusterStatus();log.info("=== HBase集群狀態 ===");log.info("活躍RegionServer數量: {}", metrics.getLiveServerMetrics().size());log.info("死亡RegionServer數量: {}", metrics.getDeadServerNames().size());// 檢查Region分布for?(Map.Entry<ServerName, ServerMetrics> entry :?metrics.getLiveServerMetrics().entrySet()) {ServerName serverName = entry.getKey();ServerMetrics serverMetrics = entry.getValue();log.info("RegionServer: {}", serverName.getServerName());log.info(" ?Region數量: {}", serverMetrics.getRegionMetrics().size());log.info(" ?請求數/秒: {}", serverMetrics.getRequestCountPerSecond());log.info(" ?讀請求數/秒: {}", serverMetrics.getReadRequestsCount());log.info(" ?寫請求數/秒: {}", serverMetrics.getWriteRequestsCount());}// 檢查是否需要負載均衡checkAndBalance(metrics);}?catch?(Exception e) {log.error("監控Region分布失敗", e);}}private?void?checkAndBalance(ClusterMetrics metrics)?{Map<ServerName, ServerMetrics> servers = metrics.getLiveServerMetrics();if?(servers.size() <?2) {return;}// 計算Region分布的標準差List<Integer> regionCounts = servers.values().stream().map(sm -> sm.getRegionMetrics().size()).collect(Collectors.toList());double?avg = regionCounts.stream().mapToInt(Integer::intValue).average().orElse(0);double?variance = regionCounts.stream().mapToDouble(count -> Math.pow(count - avg,?2)).average().orElse(0);double?stdDev = Math.sqrt(variance);// 如果標準差超過閾值，觸發負載均衡if?(stdDev >?5) {log.warn("Region分布不均衡，標準差: {}, 觸發負載均衡", stdDev);balanceCluster();}}
}

性能優化策略

1. RowKey設計

@Component
public?class?RowKeyDesigner?{/*** 用戶表RowKey設計* 格式: hash(userId)_userId*/public?String?generateUserRowKey(String userId)?{String hash = String.format("%02x", Math.abs(userId.hashCode()) %?16);return?hash +?"_"?+ userId;}/*** 訂單表RowKey設計* 格式: yyyyMM_orderId*/public?String?generateOrderRowKey(String orderId, LocalDateTime orderTime)?{String timePrefix = orderTime.format(DateTimeFormatter.ofPattern("yyyyMM"));return?timePrefix +?"_"?+ orderId;}/*** 日志表RowKey設計* 格式: yyyyMMddHH_hash(logId)_logId*/public?String?generateLogRowKey(String logId, LocalDateTime logTime)?{String timePrefix = logTime.format(DateTimeFormatter.ofPattern("yyyyMMddHH"));String hash = String.format("%04x", Math.abs(logId.hashCode()) %?65536);return?timePrefix +?"_"?+ hash +?"_"?+ logId;}/*** 反向時間戳RowKey（用于獲取最新數據）* 格式: (Long.MAX_VALUE - timestamp)_id*/public?String?generateReverseTimeRowKey(String id, LocalDateTime time)?{long?timestamp = time.toInstant(ZoneOffset.UTC).toEpochMilli();long?reverseTime = Long.MAX_VALUE - timestamp;return?String.format("%019d_%s", reverseTime, id);}
}

2. 批量操作優化

@Service
public?class?HBaseBatchService?{@Autowiredprivate?Connection connection;private?static?final?int?BATCH_SIZE =?1000;/*** 批量寫入優化*/public?void?batchWrite(String tableName, List<Map<String, Object>> dataList)?{try?(Table table = connection.getTable(TableName.valueOf(tableName))) {List<Put> puts =?new?ArrayList<>();for?(Map<String, Object> data : dataList) {Put put = createPut(data);puts.add(put);// 達到批次大小時執行寫入if?(puts.size() >= BATCH_SIZE) {table.put(puts);puts.clear();}}// 寫入剩余數據if?(!puts.isEmpty()) {table.put(puts);}}?catch?(IOException e) {log.error("批量寫入失敗", e);throw?new?RuntimeException(e);}}/*** 異步批量寫入*/@Asyncpublic?CompletableFuture<Void>?asyncBatchWrite(String tableName,?List<Map<String, Object>> dataList)?{return?CompletableFuture.runAsync(() -> {batchWrite(tableName, dataList);});}/*** 并行掃描*/public?List<Result>?parallelScan(String tableName, List<String> rowKeyRanges)?{List<CompletableFuture<List<Result>>> futures = rowKeyRanges.stream().map(range -> CompletableFuture.supplyAsync(() -> {String[] parts = range.split(",");return?scanRange(tableName, parts[0], parts[1]);})).collect(Collectors.toList());return?futures.stream().map(CompletableFuture::join).flatMap(List::stream).collect(Collectors.toList());}private?List<Result>?scanRange(String tableName, String startRow, String stopRow)?{List<Result> results =?new?ArrayList<>();try?(Table table = connection.getTable(TableName.valueOf(tableName))) {Scan scan =?new?Scan().withStartRow(Bytes.toBytes(startRow)).withStopRow(Bytes.toBytes(stopRow)).setCaching(1000) ?// 設置緩存大小.setBatch(100); ? ?// 設置批次大小try?(ResultScanner scanner = table.getScanner(scan)) {for?(Result result : scanner) {results.add(result);}}}?catch?(IOException e) {log.error("掃描范圍失敗: {} - {}", startRow, stopRow, e);}return?results;}private?Put?createPut(Map<String, Object> data)?{String rowKey = (String) data.get("rowKey");Put put =?new?Put(Bytes.toBytes(rowKey));data.forEach((key, value) -> {if?(!"rowKey".equals(key) && value !=?null) {String[] parts = key.split(":");if?(parts.length ==?2) {put.addColumn(Bytes.toBytes(parts[0]), Bytes.toBytes(parts[1]),?Bytes.toBytes(value.toString()));}}});return?put;}
}

3. 緩存策略

@Service
public?class?HBaseCacheService?{@Autowiredprivate?HBaseService hbaseService;@Autowiredprivate?RedisTemplate<String, Object> redisTemplate;private?static?final?String CACHE_PREFIX =?"hbase:";private?static?final?int?CACHE_TTL =?3600;?// 1小時/*** 帶緩存的數據獲取*/public?Result?getWithCache(String tableName, String rowKey)?{String cacheKey = CACHE_PREFIX + tableName +?":"?+ rowKey;// 先從緩存獲取Object cached = redisTemplate.opsForValue().get(cacheKey);if?(cached !=?null) {return?deserializeResult((String) cached);}// 緩存未命中，從HBase獲取Result result = hbaseService.get(tableName, rowKey);if?(!result.isEmpty()) {// 寫入緩存String serialized = serializeResult(result);redisTemplate.opsForValue().set(cacheKey, serialized, CACHE_TTL, TimeUnit.SECONDS);}return?result;}/*** 緩存預熱*/@EventListener(ApplicationReadyEvent.class)public?void?warmupCache()?{log.info("開始HBase緩存預熱...");// 預熱熱點數據List<String> hotKeys = getHotKeys();hotKeys.parallelStream().forEach(key -> {String[] parts = key.split(":");if?(parts.length ==?2) {getWithCache(parts[0], parts[1]);}});log.info("HBase緩存預熱完成，預熱數據: {} 條", hotKeys.size());}private?List<String>?getHotKeys()?{// 從配置或統計數據中獲取熱點鍵return?Arrays.asList("user_table:001","user_table:002","order_table:latest");}private?String?serializeResult(Result result)?{// 簡化實現，實際應使用更高效的序列化方式Map<String, String> data =?new?HashMap<>();result.rawCells().forEach(cell -> {String family = Bytes.toString(CellUtil.cloneFamily(cell));String qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));String value = Bytes.toString(CellUtil.cloneValue(cell));data.put(family +?":"?+ qualifier, value);});return?JSON.toJSONString(data);}private?Result?deserializeResult(String serialized)?{// 簡化實現return?null;?// 實際需要反序列化為Result對象}
}

監控和運維

1. 集群監控服務

@Service
@Slf4j
public?class?HBaseMonitoringService?{@Autowiredprivate?Admin admin;@Autowiredprivate?MeterRegistry meterRegistry;/*** 監控集群健康狀態*/@Scheduled(fixedRate =?60000)?// 1分鐘public?void?monitorClusterHealth()?{try?{ClusterMetrics metrics = admin.getClusterMetrics();// 記錄指標Gauge.builder("hbase.cluster.live_servers").register(meterRegistry, metrics.getLiveServerMetrics().size());Gauge.builder("hbase.cluster.dead_servers").register(meterRegistry, metrics.getDeadServerNames().size());Gauge.builder("hbase.cluster.regions").register(meterRegistry, metrics.getRegionCount());// 檢查異常狀態if?(!metrics.getDeadServerNames().isEmpty()) {log.error("發現死亡RegionServer: {}", metrics.getDeadServerNames());sendAlert("HBase集群異常",?"發現死亡RegionServer: "?+ metrics.getDeadServerNames());}}?catch?(Exception e) {log.error("監控集群健康狀態失敗", e);}}/*** 監控表級別指標*/@Scheduled(fixedRate =?300000)?// 5分鐘public?void?monitorTableMetrics()?{try?{List<TableName> tables = Arrays.asList(TableName.valueOf("user_table"),TableName.valueOf("order_table"),TableName.valueOf("log_table"));for?(TableName tableName : tables) {if?(admin.tableExists(tableName)) {monitorSingleTable(tableName);}}}?catch?(Exception e) {log.error("監控表指標失敗", e);}}private?void?monitorSingleTable(TableName tableName)?throws?IOException?{List<RegionInfo> regions = admin.getRegions(tableName);log.info("表 {} 監控信息:", tableName.getNameAsString());log.info(" ?Region數量: {}", regions.size());// 檢查Region大小分布Map<String, Long> regionSizes =?new?HashMap<>();long?totalSize =?0;for?(RegionInfo region : regions) {// 獲取Region大小（簡化實現）long?size = getRegionSize(region);regionSizes.put(region.getRegionNameAsString(), size);totalSize += size;}log.info(" ?總大小: {} MB", totalSize /?1024?/?1024);log.info(" ?平均Region大小: {} MB", totalSize / regions.size() /?1024?/?1024);// 檢查是否需要分割checkRegionSplit(tableName, regionSizes);}private?long?getRegionSize(RegionInfo region)?{// 簡化實現，實際需要通過JMX或其他方式獲取return?100?*?1024?*?1024;?// 100MB}private?void?checkRegionSplit(TableName tableName, Map<String, Long> regionSizes)?{long?maxSize =?1024?*?1024?*?1024L;?// 1GBregionSizes.entrySet().stream().filter(entry -> entry.getValue() > maxSize).forEach(entry -> {log.warn("Region {} 大小超過閾值: {} MB",?entry.getKey(), entry.getValue() /?1024?/?1024);// 可以觸發自動分割// splitLargeRegion(tableName, entry.getKey());});}/*** 性能指標監控*/@Scheduled(fixedRate =?120000)?// 2分鐘public?void?monitorPerformanceMetrics()?{try?{ClusterMetrics metrics = admin.getClusterMetrics();for?(Map.Entry<ServerName, ServerMetrics> entry :?metrics.getLiveServerMetrics().entrySet()) {ServerName serverName = entry.getKey();ServerMetrics serverMetrics = entry.getValue();// 記錄性能指標Tags tags = Tags.of("server", serverName.getServerName());Gauge.builder("hbase.server.request_rate").tags(tags).register(meterRegistry, serverMetrics.getRequestCountPerSecond());Gauge.builder("hbase.server.read_requests").tags(tags).register(meterRegistry, serverMetrics.getReadRequestsCount());Gauge.builder("hbase.server.write_requests").tags(tags).register(meterRegistry, serverMetrics.getWriteRequestsCount());// 檢查性能異常if?(serverMetrics.getRequestCountPerSecond() >?10000) {log.warn("RegionServer {} 請求量過高: {}/s",?serverName.getServerName(),?serverMetrics.getRequestCountPerSecond());}}}?catch?(Exception e) {log.error("監控性能指標失敗", e);}}/*** 自動故障恢復*/@EventListenerpublic?void?handleRegionServerFailure(RegionServerFailureEvent event)?{log.error("RegionServer故障: {}", event.getServerName());try?{// 等待自動恢復Thread.sleep(30000);// 檢查恢復狀態ClusterMetrics metrics = admin.getClusterMetrics();if?(metrics.getDeadServerNames().contains(event.getServerName())) {log.error("RegionServer {} 未能自動恢復，需要人工干預", event.getServerName());sendAlert("HBase故障",?"RegionServer "?+ event.getServerName() +?" 需要人工恢復");}?else?{log.info("RegionServer {} 已自動恢復", event.getServerName());}}?catch?(Exception e) {log.error("處理RegionServer故障失敗", e);}}private?void?sendAlert(String title, String message)?{// 發送告警通知（郵件、短信、釘釘等）log.error("告警: {} - {}", title, message);}
}// 自定義事件
public?class?RegionServerFailureEvent?{private?final?ServerName serverName;public?RegionServerFailureEvent(ServerName serverName)?{this.serverName = serverName;}public?ServerName?getServerName()?{return?serverName;}
}

2. 自動化運維腳本

#!/bin/bash
# hbase-ops.sh - HBase運維腳本HBASE_HOME="/opt/hbase"
ZK_QUORUM="localhost:2181"# 檢查集群狀態
check_cluster_status() {echo?"檢查HBase集群狀態..."# 檢查HMasterif?! pgrep -f?"HMaster"?> /dev/null;?thenecho?"錯誤: HMaster未運行"return?1fi# 檢查RegionServerrs_count=$(pgrep -f?"HRegionServer"?| wc -l)if?[?$rs_count?-eq 0 ];?thenecho?"錯誤: 沒有運行的RegionServer"return?1fiecho?"集群狀態正常: HMaster運行中,?$rs_count?個RegionServer運行中"return?0
}# 備份表數據
backup_table() {local?table_name=$1local?backup_dir=$2echo?"備份表?$table_name?到?$backup_dir..."$HBASE_HOME/bin/hbase org.apache.hadoop.hbase.mapreduce.Export \$table_name?$backup_dirif?[ $? -eq 0 ];?thenecho?"表?$table_name?備份成功"elseecho?"表?$table_name?備份失敗"return?1fi
}# 恢復表數據
restore_table() {local?table_name=$1local?backup_dir=$2echo?"從?$backup_dir?恢復表?$table_name..."$HBASE_HOME/bin/hbase org.apache.hadoop.hbase.mapreduce.Import \$table_name?$backup_dirif?[ $? -eq 0 ];?thenecho?"表?$table_name?恢復成功"elseecho?"表?$table_name?恢復失敗"return?1fi
}# 清理舊的WAL文件
cleanup_wal() {echo?"清理舊的WAL文件..."# 查找7天前的WAL文件find /opt/hbase/logs -name?"*.log"?-mtime +7 -deleteecho?"WAL文件清理完成"
}# 壓縮表
compact_table() {local?table_name=$1echo?"壓縮表?$table_name..."echo?"compact '$table_name'"?|?$HBASE_HOME/bin/hbase shellecho?"表?$table_name?壓縮完成"
}# 主函數
main() {case?$1?in"status")check_cluster_status;;"backup")backup_table?$2?$3;;"restore")restore_table?$2?$3;;"cleanup")cleanup_wal;;"compact")compact_table?$2;;*)echo?"用法:?$0?{status|backup|restore|cleanup|compact} [參數]"echo?" ?status ? ? ? ? ? ? ? ? ? ?- 檢查集群狀態"echo?" ?backup <table> <dir> ? ? ?- 備份表"echo?" ?restore <table> <dir> ? ? - 恢復表"echo?" ?cleanup ? ? ? ? ? ? ? ? ? - 清理WAL文件"echo?" ?compact <table> ? ? ? ? ? - 壓縮表"exit?1;;esac
}main?$@

配置文件

application.yml

spring:application:name:?hbase-sharding-demohbase:zookeeper-quorum:?localhostzookeeper-port:?2181max-connections:?100core-connections:?10management:endpoints:web:exposure:include:?health,metrics,prometheusmetrics:export:prometheus:enabled:?truelogging:level:org.apache.hadoop.hbase:?INFOcom.example.hbase:?DEBUG