dgsoft/alexandria-plsql-utils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #34 from krissco/master

Browse Source 
New s3 API for get_object_list
This commit is contained in:
mortenbra 2016-10-12 15:50:06 +02:00 committed by GitHub
commit 7e915716b9
1 changed files with 42 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
ora/amazon_aws_s3_pkg.pkb
View File

@ -533,14 +533,16 @@ begin
end get_bucket_region; end get_bucket_region;
procedure get_object_list (p_bucket_name in varchar2, procedure get_object_list (p_bucket_name in varchar2,
p_prefix in varchar2, p_prefix in varchar2,
p_max_keys in number, p_max_keys in number,
p_list out t_object_list, p_list out t_object_list,
p_more_marker in out varchar2) p_next_continuation_token in out varchar2)
as as
l_clob clob; l_clob clob;
l_xml xmltype; l_xml xmltype;
l_xml_is_truncated xmltype;
l_xml_next_continuation xmltype;
l_date_str varchar2(255); l_date_str varchar2(255);
l_auth_str varchar2(255); l_auth_str varchar2(255);
@ -548,7 +550,6 @@ as
l_header_names t_str_array := t_str_array(); l_header_names t_str_array := t_str_array();
l_header_values t_str_array := t_str_array(); l_header_values t_str_array := t_str_array();
l_count pls_integer := 0;
l_returnvalue t_object_list; l_returnvalue t_object_list;
begin begin
@ -557,7 +558,7 @@ begin
Purpose: get objects Purpose: get objects
Remarks: see http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?RESTObjectGET.html Remarks: see http://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
see http://code.google.com/p/plsql-utils/issues/detail?id=16 see http://code.google.com/p/plsql-utils/issues/detail?id=16
@ -573,6 +574,7 @@ begin
------ ---------- ------------------------------------- ------ ---------- -------------------------------------
MBR 15.01.2011 Created MBR 15.01.2011 Created
JKEMP 14.08.2012 Rewritten as private procedure, see remarks above JKEMP 14.08.2012 Rewritten as private procedure, see remarks above
KJS 06.10.2016 Modified to use newest S3 API which performs much better on large buckets. Changed for-loop to bulk operation.
*/ */
@ -594,38 +596,36 @@ begin
l_header_values.extend; l_header_values.extend;
l_header_values(3) := l_auth_str; l_header_values(3) := l_auth_str;
if p_more_marker is not null then if p_next_continuation_token is not null then
l_clob := make_request (get_url(p_bucket_name) || '?marker=' || p_more_marker || '&max-keys=' || p_max_keys || '&prefix=' || utl_url.escape(p_prefix), 'GET', l_header_names, l_header_values, null); l_clob := make_request (get_url(p_bucket_name) || '?list-type=2&continuation-token=' || utl_url.escape(p_next_continuation_token) || '&max-keys=' || p_max_keys || '&prefix=' || utl_url.escape(p_prefix), 'GET', l_header_names, l_header_values, null);
else else
l_clob := make_request (get_url(p_bucket_name) || '?max-keys=' || p_max_keys || '&prefix=' || utl_url.escape(p_prefix), 'GET', l_header_names, l_header_values, null); l_clob := make_request (get_url(p_bucket_name) || '?list-type=2&max-keys=' || p_max_keys || '&prefix=' || utl_url.escape(p_prefix), 'GET', l_header_names, l_header_values, null);
end if; end if;
if (l_clob is not null) and (length(l_clob) > 0) then if (l_clob is not null) and (length(l_clob) > 0) then
l_xml := xmltype (l_clob); l_xml := xmltype (l_clob);
check_for_errors (l_xml); check_for_errors (l_xml);
for l_rec in ( select extractValue(value(t), '*/Key', g_aws_namespace_s3_full),
select extractValue(value(t), '*/Key', g_aws_namespace_s3_full) as key, extractValue(value(t), '*/Size', g_aws_namespace_s3_full),
extractValue(value(t), '*/Size', g_aws_namespace_s3_full) as size_bytes, to_date(extractValue(value(t), '*/LastModified', g_aws_namespace_s3_full), g_date_format_xml)
extractValue(value(t), '*/LastModified', g_aws_namespace_s3_full) as last_modified bulk collect into l_returnvalue
from table(xmlsequence(l_xml.extract('//ListBucketResult/Contents', g_aws_namespace_s3_full))) t from table(xmlsequence(l_xml.extract('//ListBucketResult/Contents', g_aws_namespace_s3_full))) t;
) loop
l_count := l_count + 1; -- check if this is the last set of data or not, and set the in/out p_next_continuation_token as expected
l_returnvalue(l_count).key := l_rec.key; l_xml_is_truncated := l_xml.extract('//ListBucketResult/IsTruncated/text()', g_aws_namespace_s3_full);
l_returnvalue(l_count).size_bytes := l_rec.size_bytes;
l_returnvalue(l_count).last_modified := to_date(l_rec.last_modified, g_date_format_xml);
end loop;
-- check if this is the last set of data or not if l_xml_is_truncated is not null and l_xml_is_truncated.getStringVal = 'true' then
l_xml_next_continuation := l_xml.extract('//ListBucketResult/NextContinuationToken/text()', g_aws_namespace_s3_full);
l_xml := l_xml.extract('//ListBucketResult/IsTruncated/text()', g_aws_namespace_s3_full); if l_xml_next_continuation is not null then
p_next_continuation_token := l_xml_next_continuation.getStringVal;
if l_xml is not null and l_xml.getStringVal = 'true' then else
p_more_marker := l_returnvalue(l_returnvalue.last).key; p_next_continuation_token := null;
end if;
else
p_next_continuation_token := null;
end if; end if;
end if; end if;
p_list := l_returnvalue; p_list := l_returnvalue;
@ -638,7 +638,7 @@ function get_object_list (p_bucket_name in varchar2,
p_max_keys in number := null) return t_object_list p_max_keys in number := null) return t_object_list
as as
l_object_list t_object_list; l_object_list t_object_list;
l_more_marker varchar2(4000); l_next_continuation_token varchar2(4000);
begin begin
/* /*
@ -654,11 +654,11 @@ begin
*/ */
get_object_list ( get_object_list (
p_bucket_name => p_bucket_name, p_bucket_name => p_bucket_name,
p_prefix => p_prefix, p_prefix => p_prefix,
p_max_keys => p_max_keys, p_max_keys => p_max_keys,
p_list => l_object_list, p_list => l_object_list,
p_more_marker => l_more_marker --ignored by this function p_next_continuation_token => l_next_continuation_token --ignored by this function
); );
return l_object_list; return l_object_list;
@ -671,7 +671,7 @@ function get_object_tab (p_bucket_name in varchar2,
p_max_keys in number := null) return t_object_tab pipelined p_max_keys in number := null) return t_object_tab pipelined
as as
l_object_list t_object_list; l_object_list t_object_list;
l_more_marker varchar2(4000); l_next_continuation_token varchar2(4000);
begin begin
/* /*
@ -689,18 +689,18 @@ begin
loop loop
get_object_list ( get_object_list (
p_bucket_name => p_bucket_name, p_bucket_name => p_bucket_name,
p_prefix => p_prefix, p_prefix => p_prefix,
p_max_keys => p_max_keys, p_max_keys => p_max_keys,
p_list => l_object_list, p_list => l_object_list,
p_more_marker => l_more_marker p_next_continuation_token => l_next_continuation_token
); );
for i in 1 .. l_object_list.count loop for i in 1 .. l_object_list.count loop
pipe row (l_object_list(i)); pipe row (l_object_list(i));
end loop; end loop;
exit when l_more_marker is null; exit when l_next_continuation_token is null;
end loop; end loop;