improve data format INSERT (unfinished)

.gitignore

@@ -1,3 +1,4 @@
 *.zip
 node_modules
 test_export
+test/app_100.zip

package.json

@@ -7,18 +7,16 @@
     "url": "https://github.com/ogobrecht/plex.git"
   },
   "scripts": {
-    "prebuild": "npx ploc --in src/PLEX.pks --out README.md",
+    "prebuild":          "npx ploc --in src/PLEX.pks --out README.md",
-    "build": "node src/build.js",
+    "build":             "node src/build.js",
-    "postbuild": "echo exit | sqlplus -S /@playground @plex_install.sql",
+    "postbuild":         "echo exit | sqlplus -S /@playground @plex_install.sql",
-    "watch": "chokidar src/PLEX.pks src/PLEX.pkb src/plex_install.sql --initial -c \"npm run build\"",
+    "watch":             "chokidar src/PLEX.pks src/PLEX.pkb src/plex_install.sql --initial -c \"npm run build\"",
-    "test-ccflags": "cd src && echo exit | sqlplus -S /@playground @plex_test_ccflags.sql && cd ..",
+    "test-ccflags":      "cd test && echo exit | sqlplus -S /@playground @test_ccflags.sql         && cd ..",
-    "test-export": "cd src && echo exit | sqlplus -S /@playground @plex_test_export.sql && cd ..",
+    "test-export":       "cd test && echo exit | sqlplus -S /@playground @test_export.sql          && cd ..",
-    "test-types-table": "cd src && echo exit | sqlplus -S /@playground @plex_test_types_1_table.sql && cd ..",
+    "test-types-table":  "cd test && echo exit | sqlplus -S /@playground @test_types_1_table.sql   && cd ..",
-    "test-types-data": "cd src && echo exit | sqlplus -S /@playground @plex_test_types_2_data.sql 1000 && cd ..",
+    "test-types-data":   "cd test && echo exit | sqlplus -S /@playground @test_types_2_data.sql 1  && cd ..",
-    "test-types-export": "cd src && echo exit | sqlplus -S /@playground @plex_test_types_3_export.sql && cd ..",
+    "test-types-export": "cd test && echo exit | sqlplus -S /@playground @test_types_3_export.sql  && cd ..",
-    "test-types-import-connor": "cd src && echo exit | sqlplus -S /@playground @plex_test_types_4_import_connor.sql && cd ..",
+    "test-types-import": "cd test && echo exit | sqlplus -S /@playground @test_types_4_import.sql  && cd .."
-    "test-types-import-ottmar": "cd src && echo exit | sqlplus -S /@playground @plex_test_types_5_import_ottmar.sql && cd ..",
-    "test-types-import-sqldev": "cd src && echo exit | sqlplus -S /@playground @plex_test_types_6_import_sqldev.sql && cd .."
   },
   "devDependencies": {
     "chokidar-cli": "^2.1.0",

plex_install.sql

@@ -1739,8 +1739,8 @@ IS
         dbms_sql.define_column(v_cursor, i, v_buffer_xmltype);
       ELSIF v_desc_tab(i).col_type = c_long THEN
         dbms_sql.define_column_long(v_cursor, i);
-      ELSIF v_desc_tab(i).col_type IN (c_raw, c_long_raw, c_blob, c_bfile) THEN
+--      ELSIF v_desc_tab(i).col_type IN (c_raw, c_long_raw, c_blob, c_bfile) THEN
-        NULL; --> we ignore binary data types
+--        NULL; --> we ignore binary data types
       ELSE
         dbms_sql.define_column(v_cursor, i, v_buffer_varchar2, c_vc2_max_size);
       END IF;
@@ -1777,6 +1777,9 @@ IS
     LOOP
       EXIT WHEN dbms_sql.fetch_rows(v_cursor) = 0 OR v_data_count = p_max_rows;
       v_data_count  := v_data_count + 1;
+      if v_data_count = 1 then
+        create_header;
+      end if;
       if p_insert_all_size > 0 and mod(v_data_count, p_insert_all_size) = 1 then
         util_clob_append('insert all' || c_crlf);
       end if;
@@ -1811,8 +1814,15 @@ IS
             process_varchar2_buffer(p_quote_string => false);
           END IF;
         ELSIF v_desc_tab(i).col_type IN (c_raw, c_long_raw, c_blob, c_bfile) THEN
-            v_buffer_varchar2 := 'NULL/*Binary data type skipped - currently not supported*/';
+          dbms_sql.column_value(v_cursor, i, v_buffer_varchar2);
+          if length(v_buffer_varchar2) = 0 then
+            v_buffer_varchar2 := 'NULL/*Binary data type skipped - too large*/';
             process_varchar2_buffer(p_quote_string => false);
+          end if;
+          util_clob_append('utl_raw.cast_to_raw(q''{');
+          util_clob_append(v_buffer_varchar2);
+          util_clob_append('}'')');
+          --process_varchar2_buffer(p_quote_string => true);
         ELSE
           dbms_sql.column_value(v_cursor, i, v_buffer_varchar2);
           process_varchar2_buffer(
@@ -1848,9 +1858,9 @@ IS
         util_clob_append('select * from dual;' || c_crlf);
       end if;
       util_clob_append('commit;' || c_crlf);
+      util_clob_append('alter session set cursor_sharing = exact;' || c_crlf);
+      util_clob_append('timing stop' || c_crlf);
     end if;
-    util_clob_append('alter session set cursor_sharing = exact;' || c_crlf);
-    util_clob_append('timing stop' || c_crlf);
     util_clob_append('' || c_crlf);
   END create_footer;
 
@@ -1860,7 +1870,6 @@ BEGIN
   IF p_table_name IS NOT NULL THEN
     set_session_nls_params;
     parse_query_and_describe_columns;
-    create_header;
     create_data;
     create_footer;
     recover_session_nls_params;

src/PLEX.pkb

@@ -1098,8 +1098,8 @@ IS
         dbms_sql.define_column(v_cursor, i, v_buffer_xmltype);
       ELSIF v_desc_tab(i).col_type = c_long THEN
         dbms_sql.define_column_long(v_cursor, i);
-      ELSIF v_desc_tab(i).col_type IN (c_raw, c_long_raw, c_blob, c_bfile) THEN
+--      ELSIF v_desc_tab(i).col_type IN (c_raw, c_long_raw, c_blob, c_bfile) THEN
-        NULL; --> we ignore binary data types
+--        NULL; --> we ignore binary data types
       ELSE
         dbms_sql.define_column(v_cursor, i, v_buffer_varchar2, c_vc2_max_size);
       END IF;
@@ -1136,6 +1136,9 @@ IS
     LOOP
       EXIT WHEN dbms_sql.fetch_rows(v_cursor) = 0 OR v_data_count = p_max_rows;
       v_data_count  := v_data_count + 1;
+      if v_data_count = 1 then
+        create_header;
+      end if;
       if p_insert_all_size > 0 and mod(v_data_count, p_insert_all_size) = 1 then
         util_clob_append('insert all' || c_crlf);
       end if;
@@ -1170,8 +1173,15 @@ IS
             process_varchar2_buffer(p_quote_string => false);
           END IF;
         ELSIF v_desc_tab(i).col_type IN (c_raw, c_long_raw, c_blob, c_bfile) THEN
-            v_buffer_varchar2 := 'NULL/*Binary data type skipped - currently not supported*/';
+          dbms_sql.column_value(v_cursor, i, v_buffer_varchar2);
+          if length(v_buffer_varchar2) = 0 then
+            v_buffer_varchar2 := 'NULL/*Binary data type skipped - too large*/';
             process_varchar2_buffer(p_quote_string => false);
+          end if;
+          util_clob_append('utl_raw.cast_to_raw(q''{');
+          util_clob_append(v_buffer_varchar2);
+          util_clob_append('}'')');
+          --process_varchar2_buffer(p_quote_string => true);
         ELSE
           dbms_sql.column_value(v_cursor, i, v_buffer_varchar2);
           process_varchar2_buffer(
@@ -1207,9 +1217,9 @@ IS
         util_clob_append('select * from dual;' || c_crlf);
       end if;
       util_clob_append('commit;' || c_crlf);
+      util_clob_append('alter session set cursor_sharing = exact;' || c_crlf);
+      util_clob_append('timing stop' || c_crlf);
     end if;
-    util_clob_append('alter session set cursor_sharing = exact;' || c_crlf);
-    util_clob_append('timing stop' || c_crlf);
     util_clob_append('' || c_crlf);
   END create_footer;
 
@@ -1219,7 +1229,6 @@ BEGIN
   IF p_table_name IS NOT NULL THEN
     set_session_nls_params;
     parse_query_and_describe_columns;
-    create_header;
     create_data;
     create_footer;
     recover_session_nls_params;

test/test_export.sql

@@ -21,6 +21,9 @@ BEGIN
 END;
 /
 
+prompt Delete old zip file from previous test:
+host del app_100.zip
+
 set termout off
 spool "app_100.zip.base64"
 print zip

test/test_types_4_import.sql

@@ -0,0 +1,21 @@
+prompt Truncate table plex_test_multiple_datatypes
+TRUNCATE TABLE plex_test_multiple_datatypes;
+
+-- Script generated by PLEX version 2.2.0.3 - more infos here: https://github.com/ogobrecht/plex
+-- Performance Hacks by Connor McDonald: https://connor-mcdonald.com/2019/05/17/hacking-together-faster-inserts/
+prompt Insert into PLEX_TEST_MULTIPLE_DATATYPES
+timing start inserts
+set define off feedback off
+alter session set cursor_sharing = force;
+alter session set nls_numeric_characters = '.,';
+alter session set nls_date_format = 'yyyy-mm-dd hh24:mi:ss';
+alter session set nls_timestamp_format = 'yyyy-mm-dd hh24:mi:ssxff';
+alter session set nls_timestamp_tz_format = 'yyyy-mm-dd hh24:mi:ssxff tzr';
+insert all
+  into PLEX_TEST_MULTIPLE_DATATYPES(PTMD_ID,PTMD_VARCHAR,PTMD_CHAR,PTMD_INTEGER,PTMD_NUMBER,PTMD_NUMBER_X_5,PTMD_NUMBER_20_5,PTMD_FLOAT,PTMD_FLOAT_SIZE_30,PTMD_XMLTYPE,PTMD_CLOB,PTMD_BLOB,PTMD_DATE,PTMD_TIMESTAMP,PTMD_TIMESTAMP_TZ,PTMD_TIMESTAMP_LTZ,PTMD_INTERVAL_DAY_TO_SECOND,PTMD_INTERVAL_YEAR_TO_MONTH,PTMD_LONG) values (15772,'rfMIEUOgHNXa','p',541774245734,622292439814,577624.30603,421867624225803.93867,413691607976146553450595.342509837628,637946915900000000000000,'<dummy>Dummy XML for API method get_a_row: eHkyfSKLErudVVekInOtQjkSEAhdpMOOCZpLfSPZFbljsFzxPvwmESyjHScVfYaHRkpBouJOOA</dummy>
+','Dummy clob for API method get_a_row: JokPUveQueMyKsRhKrqgWMakgAJMevZwtCODDxsQJlYRJCWLfkqkbbNqotbyrDMnhbBNp',utl_raw.cast_to_raw(q'{Dummy blob for API method get_a_row: xUdOxgOxwMkzwFXmmhwqCuBFEqMaezMTgIznRpYJCZtY}'),'1915-12-04 00:00:00','2020-11-10 20:25:28.522977','2020-11-10 20:25:28.522983 +00:00','2020-11-10 21:25:28.522985',NULL,NULL,NULL)
+select * from dual;
+commit;
+alter session set cursor_sharing = exact;
+timing stop
+