从数据库中提取BLOB获取速度较慢

Question

我们一直在运行存储过程，每天大概在过去的5年中从数据库中提取BLOBS。我们通常每晚提取大约25个文件，其中大部分文件大小约为500KB，接近10,000KB。

这个过程从来都不是最快的，但是在我们移动数据中心之后，这个过程可能需要12个小时。当你只提取〜55MB左右时，这本身就令人震惊。我们邀请所有相关团队了解Oracle的性能，磁盘I/O等，他们声称一切都是完美的。

我一直在阅读有关UTL_FILE和DBMS_LOB.read，看到人们谈论重置每个循环等后pos说实话，我似乎无法找出任何这意味着，和一般的共识是，有是达到相同结果的更好方法。

不幸的是，我们没有重构这个的自由，所以任何人都可以看到我们的程序明显错误吗？我只是在努力处理一些我不完全理解的东西，那些维护我们的基础设施的人会将这一切都归咎于这些代码，并为此付出代价。

CREATE OR REPLACE PROCEDURE PKG_EXTRACT (l_brand IN VARCHAR2) AS 

    l_file  UTL_FILE.FILE_TYPE; 
    l_buffer RAW(32767); 
    l_amount BINARY_INTEGER := 32767; 
    l_pos  INTEGER; 
    l_blob  BLOB; 
    l_blob_len INTEGER; 
    x NUMBER; 
    l_file_name VARCHAR2(200); 
    l_count  INTEGER := 1; 
    v_code NUMBER; 
    v_errm VARCHAR2(64); 
log_file  UTL_FILE.FILE_TYPE; 
rec_num number; 

BEGIN 
DECLARE 
    CURSOR extract_cur IS 
     SELECT DATA, BIN_NAME 
     FROM STAGING 
     WHERE UPPER(EXTRACTED)='N'; 

BEGIN 
log_file := UTL_FILE.fopen('DATA_DOWNLOAD_DIR','pkg_extract.log','a', 32767); 

UTL_FILE.put_line(log_file,'Logging is being done in 24 hours format - V1.5 ',TRUE); 
UTL_FILE.put_line(log_file,'Extract procedure started on Date-Time = '|| TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'),TRUE); 


select count(1) into rec_num from staging; 

UTL_FILE.put_line(log_file,'Total Number of records found = ' || rec_num , TRUE); 
select count(1) into rec_num from staging where UPPER(EXTRACTED)='N'; 
UTL_FILE.put_line(log_file,'Total Number of records matching criteria = ' || rec_num , TRUE); 

    dbms_output.put_line('Loop through records and write them to file'); 
    FOR extract_rec IN extract_cur 
    LOOP 

     l_pos := 1; 
     l_blob := extract_rec.DATA; 
     l_blob_len := DBMS_LOB.getlength(l_blob); 

     -- Save blob length. 
     x := l_blob_len; 

     l_file_name := extract_rec.BIN_NAME ; 

     -- Open the destination file. 
     dbms_output.put_line('Open the destination file:- ' || l_file_name); 
     l_file := UTL_FILE.fopen('DATA_DOWNLOAD_DIR',l_file_name,'wb', 32767); 
     dbms_output.put_line('File opened'); 

     -- Read chunks of the BLOB and write them to the file until complete. 
     dbms_output.put_line('l_pos:- ' || l_pos); 
     dbms_output.put_line('l_blob_len:- ' || l_blob_len); 
     WHILE l_pos <= l_blob_len 
     LOOP 
     dbms_output.put_line('DBMS_LOB.read from position: ' || l_pos); 
     DBMS_LOB.read(l_blob, l_amount, l_pos, l_buffer); 
     dbms_output.put_line('UTL_FILE.put_raw'); 
     UTL_FILE.put_raw(l_file, l_buffer, TRUE); 
     dbms_output.put_line('Written ' || l_amount || ' bytes of data starting at position: ' || l_pos); 

     -- Set the start position for the next cut. 
     l_pos := l_pos + l_amount; 

    --updating the extract field 

     dbms_output.put_line(extract_rec.BIN_NAME); 

     END LOOP; 

     l_count := l_count + 1; 
     -- Close the file. 
     dbms_output.put_line('Close the file:- ' || l_file_name); 
     UTL_FILE.fclose(l_file); 

     update staging set extracted='Y', extract_timestamp=sysdate where bin_name=extract_rec.BIN_NAME; 
     commit; 

    END LOOP; 

UTL_FILE.put_line(log_file,'Extract procedure Completed on Date-Time = '|| TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'),TRUE); 

IF UTL_FILE.is_open(log_file) THEN 
     UTL_FILE.fclose(log_file); 
end if; 
END; 

EXCEPTION 
    WHEN OTHERS THEN 
    v_code := SQLCODE; 
    v_errm := SUBSTR(SQLERRM, 1, 64); 
    dbms_output.put_line('Error code ' || v_code || ': ' || v_errm); 
UTL_FILE.put_line(log_file,'--------------------------------------' ,TRUE); 
UTL_FILE.put_line(log_file,'Error Occurred while executing '||'Error code ' || v_code || ': ' || v_errm ,TRUE); 
UTL_FILE.put_line(log_file,'Extract procedure Completed with errors - '|| TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'),TRUE); 
UTL_FILE.put_line(log_file,'--------------------------------------' ,TRUE); 

    -- Close the file if something goes wrong. 
    IF UTL_FILE.is_open(l_file) THEN 

     UTL_FILE.fclose(l_file); 

IF UTL_FILE.is_open(log_file) THEN 
     UTL_FILE.fclose(log_file); 
end if; 
    END IF; 
    RAISE; 
END; 
/

---

编辑

为CURSOR extract_cur执行计划。

Plan hash value: 3428151562 
----------------------------------------------------------------------------- 
| Id | Operation   | Name | Rows | Bytes | Cost (%CPU)| Time  | 
----------------------------------------------------------------------------- 
| 0 | SELECT STATEMENT |   |  6 | 678 |  9 (0)| 00:00:01 | 
|* 1 | TABLE ACCESS FULL| STAGING |  6 | 678 |  9 (0)| 00:00:01 | 
----------------------------------------------------------------------------- 
Predicate Information (identified by operation id): 
--------------------------------------------------- 
    1 - filter(UPPER("S"."EXTRACTED")='N') 

Answer 1

行，所以我改变了你的代码，尽量避免这些东西select count(1) into rec_num from staging因为Oracle如何在这里将它转换PL/SQL到SQL和SQL PL/SQL所以更好地利用光标，这里是改变的代码。并尝试添加索引create index extracted_idx on staging(extracted asc);所以这里是修改的代码：我不知道它是否编译，因为我无法检查它，但它应该工作。

CREATE OR REPLACE PROCEDURE PKG_EXTRACT(l_brand IN VARCHAR2) AS 

    l_file  UTL_FILE.FILE_TYPE; 
    l_buffer RAW(32767); 
    l_amount BINARY_INTEGER := 32767; 
    l_pos  INTEGER; 
    l_blob  BLOB; 
    l_blob_len INTEGER; 
    x   NUMBER; 
    l_file_name VARCHAR2(200); 
    l_count  INTEGER := 1; 
    v_code  NUMBER; 
    v_errm  VARCHAR2(64); 
    log_file UTL_FILE.FILE_TYPE; 
    rec_num  number; 

    CURSOR extract_cur IS 
    SELECT DATA, 
       BIN_NAME 
      FROM STAGING 
     WHERE EXTRACTED = 'N'; 

    cursor c_count is 
    select count(1) cnt 
      from staging; 

    cursor c_rec_num is 
    select count(1) cnt 
     from staging 
     where EXTRACTED = 'N'; 

BEGIN 
    log_file := UTL_FILE.fopen('DATA_DOWNLOAD_DIR', 
          'pkg_extract.log', 
          'a', 
          32767); 

    UTL_FILE.put_line(log_file, 
        'Logging is being done in 24 hours format - V1.5 ', 
        TRUE); 
    UTL_FILE.put_line(log_file, 
        'Extract procedure started on Date-Time = ' || 
        TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'), 
        TRUE); 

    open c_cnt; 
    fetch c_cnt into rec_num; 
    close c_cnt; 

    open c_rec_num; 
    fetch c_rec_num into rec_num; 
    close c_rec_num; 

    UTL_FILE.put_line(log_file, 
        'Total Number of records found = ' || rec_num, 
        TRUE); 
    --dont know why you doing this again 
    open c_rec_num; 
    fetch c_rec_num into rec_num; 
    close c_rec_num; 

    UTL_FILE.put_line(log_file, 
        'Total Number of records matching criteria = ' || 
        rec_num, 
        TRUE); 

    FOR extract_rec IN extract_cur LOOP 

    l_pos  := 1; 
    l_blob  := extract_rec.DATA; 
    l_blob_len := DBMS_LOB.getlength(l_blob); 

    -- Save blob length. 
    x := l_blob_len; 

    l_file_name := extract_rec.BIN_NAME; 

    -- Open the destination file. 

    l_file := UTL_FILE.fopen('DATA_DOWNLOAD_DIR', l_file_name, 'wb', 32767); 

    -- Read chunks of the BLOB and write them to the file until complete. 

    WHILE l_pos <= l_blob_len LOOP 

     DBMS_LOB.read(l_blob, l_amount, l_pos, l_buffer); 

     UTL_FILE.put_raw(l_file, l_buffer, TRUE); 

     -- Set the start position for the next cut. 
     l_pos := l_pos + l_amount; 

    --updating the extract field 

    END LOOP; 

    l_count := l_count + 1; 
    -- Close the file. 
    UTL_FILE.fclose(l_file); 

    update staging 
     set extracted = 'Y', 
       extract_timestamp = sysdate 
    where bin_name = extract_rec.BIN_NAME; 
    commit; 

    END LOOP; 

    UTL_FILE.put_line(log_file, 
        'Extract procedure Completed on Date-Time = ' || 
        TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'), 
        TRUE); 

    IF UTL_FILE.is_open(log_file) THEN 
    UTL_FILE.fclose(log_file); 
    end if; 
END; 

EXCEPTION 
WHEN OTHERS THEN 
     v_code := SQLCODE; v_errm := SUBSTR(SQLERRM, 1, 64); 
     UTL_FILE.put_line(log_file, '--------------------------------------', TRUE); 
     UTL_FILE.put_line(log_file, 'Error Occurred while executing ' || 'Error code ' || v_code || ': ' || v_errm, TRUE); 
     UTL_FILE.put_line(log_file, 'Extract procedure Completed with errors - ' || TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'), TRUE); 
     UTL_FILE.put_line(log_file, '--------------------------------------', TRUE); 

    -- Close the file if something goes wrong. 
    IF UTL_FILE.is_open(l_file) THEN 

    UTL_FILE.fclose(l_file); 

     IF UTL_FILE.is_open(log_file) THEN 
      UTL_FILE.fclose(log_file); 
     end if; 
    END IF; 
END; 
/