完整腳本(自行替換相關變量)

user_ol=`sh conf/getInfo.sh 'Oracle' 'user' conf/databases.conf`
password_ol=`sh conf/getInfo.sh 'Oracle' 'password' conf/databases.conf`
dblink_ol=`sh conf/getInfo.sh 'Oracle' 'dblink' conf/databases.conf`
current_date=$(date +"%Y%m%d%H%M%S")
table_name=""
hdfs_path="/user/yarn/"${table_name}${current_date}echo "參數成功"${current_date}
#where $CONDITIONS 必須有
sql='
where $CONDITIONS
'
echo "開始執行sqoop"sqoop import -Dorg.apache.sqoop.splitter.allow_text_splitter=true \
--connect ${dblink_ol} \
--username ${user_ol} \
--password ${password_ol} \
--query "${sql}" \
--target-dir ${hdfs_path} \
--delete-target-dir \
--split-by card_no \
-m 2if [ $? -ne 0 ]; thenecho "導出到hdfs失敗----------------------"exit -1 
fiecho "開始導入到doris"
#broker_name
user=`sh conf/getInfo.sh 'Doris-Pro' 'user' conf/databases.conf`
password=`sh conf/getInfo.sh 'Doris-Pro' 'password' conf/databases.conf`
dblink=`sh conf/getInfo.sh 'Doris-Pro' 'dblink' conf/databases.conf`
target_table_name="ods_cme_"${table_name}
echo "加載路徑:"${hdfs_path}
lable_name=${target_table_name}"_"${current_date}#nameservice1 分布式hdfs集群名稱,單機選IP形式
sql="
LOAD LABEL ${lable_name}
(DATA INFILE('hdfs://nameservice1${hdfs_path}/part*')INTO TABLE ${target_table_name}COLUMNS TERMINATED BY ','            
) 
with BROKER worker3
PROPERTIES
('timeout'='1200','max_filter_ratio'='0.1'
)
"
mysql -h fe_ip -P9030 -u${user} -p${password} -D offline -e "truncate table ${target_table_name}"
echo "清空表成功"
mysql -h fe_id -P9030 -u${user} -p${password} -D offline -e "${sql}"if [ $? -ne 0 ]; thenecho "創建broker任務失敗--------------------"exit -1  # 可以選擇退出腳本或執行其他操作
fi# broker 任務檢查
sh conf/broker_state.sh ${label} ${user} ${password}if [ $? -ne 0 ]; thenecho "任務失敗--------------------"exit -1  # 可以選擇退出腳本或執行其他操作
fi

配置文件conf

# 文本輸出為base64的形式
[Doris]
user_name=_=用戶名
password=_=密碼
dblink=_=jdbcurl

配置文件解析腳本

#!/bin/bash
param=`awk -F '=_=' '/\['"$1"'\]/{a=1}a==1&&$1~/'"$2"'/{gsub(/[[:blank:]]*/,"",$2);printf("%s\t",$2);exit;}' $3`
echo $(echo -n ${param} | base64 --decode)

轉base64腳本

#!/bin/bash
# 要編碼的文本
text=$1
# 編碼文本
encoded_text=$(echo -n "${text}" | base64 -w 0)
# 輸出編碼結果
echo "編碼結果:${encoded_text}"
# 輸出編碼欠的結果
echo "解碼結果:"$(echo -n ${encoded_text} | base64 --decode)