生產有一張日志表,數據量很大,需要按月進行存儲,存儲過程如下:
CREATE OR REPLACE PROCEDURE NEWLOG4_SUB_TABLE
IStable_name1 VARCHAR2(50);create_table_sql VARCHAR2(4000);insert_data_sql VARCHAR2(4000);delete_data_sql VARCHAR2(4000);
-- v_exists INT:=0;v_exists NUMBER (10, 0);
BEGINSELECT 'NEWLOG4_' || TO_CHAR(ADD_MONTHS(SYSDATE, -1), 'YYYYMM') INTO table_name1 FROM DUAL;select count(1) into v_exists from user_tables where table_name=UPPER(table_name1);--dbms_output.put_line(sname);--dbms_output.put_line(table_name1);if (v_exists <1)then-- dbms_output.put_line(sname);create_table_sql := 'create table ' || table_name1 || ' (autudt TIMESTAMP(6),authentype VARCHAR2(2000),userid VARCHAR2(2000),orgid VARCHAR2(2000),org2id VARCHAR2(2000),realname VARCHAR2(2000),success VARCHAR2(2000),idpname VARCHAR2(2000),idpip VARCHAR2(2000),vistorip VARCHAR2(2000),vistorbrowser VARCHAR2(2000),spid VARCHAR2(2000),spurl VARCHAR2(2000),info VARCHAR2(2000),autdesc VARCHAR2(2000),taketime VARCHAR2(2000),orgnamefullpath VARCHAR2(2000),ines INTEGER default 0,logid VARCHAR2(32),inputaccount VARCHAR2(2000),channel NUMBER(32),pushstate NUMBER(2) default 0,appid VARCHAR2(50))';EXECUTE IMMEDIATE create_table_sql;commit;end if;--將FATHER_TABLE表中取上月記錄 添加到新創建的分表中。insert_data_sql := 'INSERT INTO ' || table_name1 || ' SELECT * FROM NEWLOG4 WHERE autudt <(systimestamp - NUMTODSINTERVAL(30,''day'')) ';EXECUTE IMMEDIATE insert_data_sql;--刪除FATHER_TABLE表中時間在上個月范圍內的所有數據delete_data_sql := 'DELETE FROM NEWLOG4 WHERE autudt <(systimestamp-NUMTODSINTERVAL(30,''day''))';EXECUTE IMMEDIATE delete_data_sql;COMMIT;--EXCEPTION--WHEN OTHERS THEN-- ROLLBACK;
END NEWLOG4_SUB_TABLE;
create or replace procedure pro_exelog2 is
datetime INTEGER :=30; --Storage duration
v_name varchar2(40);
v_lastmouth varchar2(40);
v_count number:=0;
v_tbrecordname varchar2(10) :='newlog2_';
v_table varchar2(4000);
v_exists INT:=0;
beginselect tb_name,v_tbrecordname||to_char(sysdate-datetime,'yyyymm') into v_name,v_lastmouth from tb_record where type =2 and status=1;select count(1) into v_count from tb_record where type =2 and tb_name=UPPER(v_lastmouth);if v_count=0 thenbeginupdate tb_record set status=0 where type =2; -- erase statusinsert into tb_record(type,tb_name,status) values(2,UPPER(v_lastmouth),1); --insert new table recordend;end if;select count(1) into v_exists from user_tables where table_name=UPPER(v_lastmouth);--create new table when time > 30 daysif v_exists!=1 thenbeginv_table:='create table '||v_name||' (AUTUDT TIMESTAMP(6),AUTHENTYPE VARCHAR2(2000),USERID VARCHAR2(2000),ORGID VARCHAR2(800),ORG2ID VARCHAR2(800),REALNAME VARCHAR2(800),SUCCESS VARCHAR2(20),IDPNAME VARCHAR2(2000),IDPIP VARCHAR2(2000),VISTORIP VARCHAR2(2000),VISTORBROWSER VARCHAR2(2000),SPID VARCHAR2(2000),SPURL VARCHAR2(2000),INFO VARCHAR2(2000),AUTDESC VARCHAR2(2000),TAKETIME VARCHAR2(2000),ORGNAMEFULLPATH VARCHAR2(2000))';EXECUTE IMMEDIATE v_table;commit;end;end if;v_table:='insert into '||v_lastmouth||' select * from newlog2 where autudt <(sysdate-'||datetime||')'; --insert new data from old tableEXECUTE IMMEDIATE v_table;commit;delete from newlog2 where autudt <(sysdate-datetime); --delete old datacommit;
end pro_exelog2;
create or replace procedure pro_exelog1 is
datetime INTEGER :=30;
v_name varchar2(40);
v_lastmouth varchar2(40);
v_count int:=0;
v_table varchar2(4000);
v_exists INT:=0;
v_tbrecordname varchar2(10) :='newlog1_';
beginselect tb_name ,v_tbrecordname||to_char(sysdate-datetime,'yyyymm') into v_name,v_lastmouth from tb_record where type =1 and status=1;select count(1) into v_count from tb_record where type =1 and tb_name=UPPER(v_lastmouth);if v_count=0 thenbeginupdate tb_record set status=0 where type=1; -- erase statusinsert into tb_record(type,tb_name,status) values(1,UPPER(v_lastmouth),1); --insert new table recordend;end if;select count(1) into v_exists from user_tables where table_name=UPPER(v_lastmouth);--create new table when time > 30 daysif v_exists!=1 thenbeginv_table:='create table '||v_lastmouth||' (OPDT TIMESTAMP(6),OPUSERID VARCHAR2(32),OPUSERIP VARCHAR2(20),OPTYPE VARCHAR2(20),MAINOBJECTID VARCHAR2(32), MAINOBJECTTYPE VARCHAR2(20),MAINACTION VARCHAR2(50),BEFOREACTIONOBJECTJSONSTRING CLOB,AFTERACTIONOBJECTJSONSTRING CLOB,EXCCUTESTATUS VARCHAR2(2000),ERRORDESC VARCHAR2(2000),MAINOBJECTORG VARCHAR2(32),LOG_ID VARCHAR2(32) not null,PLATFORM VARCHAR2(20))';EXECUTE IMMEDIATE v_table;commit;end;end if;v_table:='insert into '||v_lastmouth||' select * from newlog1 where opdt <(sysdate-'||datetime||')'; --insert new data from old tableEXECUTE IMMEDIATE v_table;commit;delete from newlog1 where opdt <(sysdate-datetime); --delete old datacommit;
end pro_exelog1;
create or replace
PROCEDURE PROC_CREATE_SUB_TABLE
IStable_name VARCHAR2(50);create_table_cursor NUMBER(10);create_table_sql VARCHAR2(1000);insert_data_sql VARCHAR2(1000);delete_data_sql VARCHAR2(1000);v_exists INT:=0;
BEGIN--生成分表的表名。SELECT 'NEWLOG4_' || TO_CHAR(ADD_MONTHS(SYSDATE, -1), 'YYYYMM') INTO table_name FROM DUAL;select count(1) into v_exists from user_tables where table_name=UPPER(table_name);if v_exists!=1 then create_table_cursor := DBMS_SQL.OPEN_CURSOR;--打開游標--拼出創建表的SQL語句,并執行。create_table_sql := 'create table ' || table_name || ' (autudt TIMESTAMP(6),authentype VARCHAR2(2000),userid VARCHAR2(2000),orgid VARCHAR2(2000),org2id VARCHAR2(2000),realname VARCHAR2(2000),success VARCHAR2(2000),idpname VARCHAR2(2000),idpip VARCHAR2(2000),vistorip VARCHAR2(2000),vistorbrowser VARCHAR2(2000),spid VARCHAR2(2000),spurl VARCHAR2(2000),info VARCHAR2(2000),autdesc VARCHAR2(2000),taketime VARCHAR2(2000),orgnamefullpath VARCHAR2(2000),ines INTEGER default 0,logid VARCHAR2(32),inputaccount VARCHAR2(2000),channel NUMBER(32),pushstate NUMBER(2) default 0)';DBMS_SQL.PARSE(create_table_cursor, create_table_sql, DBMS_SQL.V7);DBMS_SQL.CLOSE_CURSOR(create_table_cursor);end if;--將FATHER_TABLE表中取上月記錄 添加到新創建的分表中。insert_data_sql := 'INSERT INTO ' || table_name || ' SELECT * FROM NEWLOG4 WHERE autudt <(systimestamp + NumToYMInterval(-1, 'MONTH')) ';v_table:='insert into '||v_lastmouth||' select * from newlog2 where autudt <(sysdate-'||datetime||')';EXECUTE IMMEDIATE insert_data_sql;--刪除FATHER_TABLE表中時間在上個月范圍內的所有數據delete_data_sql := 'DELETE FROM NEWLOG4 WHERE autudt <(systimestamp + NumToYMInterval(-1, 'MONTH')) ';EXECUTE IMMEDIATE delete_data_sql;COMMIT;EXCEPTIONWHEN OTHERS THENROLLBACK;
END PROC_CREATE_SUB_TABLE;
CREATE OR REPLACE PROCEDURE NEWLOG4_history_table
IS-- table_name1 VARCHAR2(50);-- create_table_sql VARCHAR2(4000);-- insert_data_sql VARCHAR2(4000);-- delete_data_sql VARCHAR2(4000);
-- v_exists INT:=0;-- v_exists NUMBER (10, 0);CURSOR cur ISselect * from NEWLOG4 t where autudt <(systimestamp-NUMTODSINTERVAL(30,'day'))and pushstate='3';TYPE rec IS TABLE OF NEWLOG4%ROWTYPE;recs rec;
BEGIN
OPEN cur;
WHILE (TRUE) LOOP
FETCH cur BULK COLLECT
INTO recs LIMIT 5000;
FORALL i IN 1 .. recs.COUNT
INSERT INTO NEWLOG4_DAY_INTERVAL_PARTITION VALUES recs (i);COMMIT;EXIT WHEN cur%NOTFOUND;END LOOP;CLOSE cur;--EXCEPTION--WHEN OTHERS THEN-- ROLLBACK;
END NEWLOG4_history_table;
CREATE OR REPLACE PROCEDURE NEWLOG4_day_TABLE (delete_date in varchar2)
ISinsert_data_sql VARCHAR2(4000);delete_data_sql VARCHAR2(4000);CURSOR cur IS
select * from NEWLOG4 t where TO_CHAR(t.autudt,'YYYY-MM-DD') = delete_date and pushstate='3';
TYPE rec IS TABLE OF NEWLOG4%ROWTYPE;
recs rec;
BEGIN
OPEN cur;
WHILE (TRUE) LOOP
FETCH cur BULK COLLECT
INTO recs LIMIT 5000;
FORALL i IN 1 .. recs.COUNT
--EXECUTE IMMEDIATE insert_data_sql;
INSERT INTO NEWLOG4_DAY_INTERVAL_PARTITION VALUES recs (i);COMMIT;EXIT WHEN cur%NOTFOUND;END LOOP;CLOSE cur;--EXCEPTION--WHEN OTHERS THEN-- ROLLBACK;
END NEWLOG4_day_TABLE;
CREATE OR REPLACE PROCEDURE NEWLOG4_SUB_test (delete_date in varchar2)
IS-- table_name1 VARCHAR2(50);-- create_table_sql VARCHAR2(4000);insert_data_sql VARCHAR2(4000);delete_data_sql VARCHAR2(4000);
-- v_exists INT:=0;-- v_exists NUMBER (10, 0);CURSOR cur IS
select * from NEWLOG4 t where TO_CHAR(t.autudt,'YYYY-MM-DD') = delete_date and pushstate='3';
TYPE rec IS TABLE OF NEWLOG4%ROWTYPE;
recs rec;
BEGIN--將FATHER_TABLE表中取上月記錄 添加到新創建的分表中。-- insert_data_sql := 'INSERT INTO ' || table_name1 || ' SELECT * FROM NEWLOG4 WHERE autudt <(systimestamp - NUMTODSINTERVAL(30,''day'')) and pushstate=''3''';-- EXECUTE IMMEDIATE insert_data_sql;--insert_data_sql :='INSERT INTO '||history_table|| 'VALUES recs (i)'
OPEN cur;
WHILE (TRUE) LOOP
FETCH cur BULK COLLECT
INTO recs LIMIT 5000;
FORALL i IN 1 .. recs.COUNT
--EXECUTE IMMEDIATE insert_data_sql;
INSERT INTO NEWLOG4_202103_10 VALUES recs (i);COMMIT;EXIT WHEN cur%NOTFOUND;END LOOP;CLOSE cur;--insert_data_sql :='alter session enable parallel dml';--EXECUTE IMMEDIATE insert_data_sql;-- delete_data_sql :='delete /*+parallel(12)*/ from NEWLOG4 nologging where TO_CHAR(autudt,''YYYY-MM-DD'') =''2021-03-10''and pushstate=''3''';--delete_data_sql :='delete /*+parallel(12)*/ from NEWLOG4 nologging where TO_CHAR(autudt,''YYYY-MM-DD'') =' ||delete_date|| 'and pushstate=''3''';--EXECUTE IMMEDIATE delete_data_sql;--COMMIT;--EXCEPTION--WHEN OTHERS THEN-- ROLLBACK;
END NEWLOG4_SUB_test;
2.alter table tab1 nologging;insert /*+ append */ into tab1 select * from tab2;commit;alter table tab1 logging;該方法會使得產生arch大大減少,并且在一定程度上提高時間,根據經驗,千萬級的數據可在45分鐘內完成。但是請注意,該方法適合單進程的串行方式,如果當有多個進程同時運行時,后發起的進程會有enqueue的等待。注意此方法千萬不能dataguard上用(不過要是在database已經force logging那也是不怕的,呵呵)!!3. insert into tab1 select /*+ parallel */ * from tab2;commit;對于select之后的語句是全表掃描的情況,我們可以加parallel的hint來提高其并發,這里需要注意的是最大并發度受到初始化參數parallel_max_servers的限制,并發的進程可以通過v$px_session查看,或者ps -ef |grep ora_p查看。4. alter session enable parallel dml;insert /*+ parallel */ into tab1 select * from tab2;commit;與方法2相反,并發的insert,尚未比較和方法2哪個效率更高(偶估計是方法2快),有測試過的朋友歡迎補充。5.insert into tab1 select * from tab2 partition (p1);insert into tab1 select * from tab2 partition (p2);insert into tab1 select * from tab2 partition (p3);insert into tab1 select * from tab2 partition (p4);對于分區表可以利用tab1進行多個進程的并發insert,分區越多,可以啟動的進程越多。我曾經試過insert 2.6億行記錄的一個表,8個分區,8個進程,如果用方法2,單個進程完成可能要40分鐘,但是由于是有8個分區8個進程,后發進程有enqueue,所以因此需要的時間為40分鐘×8;但是如果用方法5,雖然單個進程需要110分鐘,但是由于能夠并發進程執行,所以總共需要的時間就約為110分鐘了。6.DECLARETYPE dtarray IS TABLE OF VARCHAR2(20) INDEX BY BINARY_INTEGER;v_col1 dtarray;v_col2 dtarray;v_col3 dtarray;BEGINSELECT col1, col2, col3 BULK COLLECTINTO v_col1, v_col2, v_col3FROM tab2;FORALL i IN 1 .. v_col1.COUNTinsert into tab1 WHERE tab1.col1 = v_col1;END;用批量綁定(bulk binding)的方式。當循環執行一個綁定變量的sql語句時候,在PL/SQL 和SQL引擎(engines)中,會發生大量的上下文切換(context switches)。使用bulk binding,能將數據批量的從plsql引擎傳到sql引擎,從而減少上下文切換過程,提升效率。該方法比較適合于在線處理,不必停機
————————————————7.sqlplus -s user/pwd< runlog.txtset copycommit 2;set arraysize 5000;copy from user/pwd@sid -to user/pwd@sid -insert tab1 using select * from tab2;exitEOF用copy的方法進行插入,注意此處insert沒有into關鍵字。該方法的好處是可以設置copycommit和arrarysize來一起控制commit的頻率,上面的方法是每10000行commit一次
?