vitalif
/
mdtest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #362 from hpc/s3-improve 

This patch adds some minor improvements to the error handling in S3-libS3 when using a single bucket for the deletion operation.

The new environment variable "S3LIB_DELETE_HEURISTICS" allows controlling at which file size of the initial fragment the list_bucket operation is executed. Otherwise, for each deleted file/directory the list_bucket operation is invoked which is very inefficient. Note that S3-libs3 intends to be an "optimal"/best case performance for S3 anyway.
master
Julian Kunkel 2021-04-23 12:24:21 +02:00 committed by GitHub
parent 11a81e42ca be80b4e618
commit d19db1085a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 32 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/aiori-S3-libs3.c
View File

@ -171,7 +171,7 @@ static void S3_Sync(aiori_mod_opt_t * options)
static S3Status S3ListResponseCallback(const char *ownerId, const char *ownerDisplayName, const char *bucketName, int64_t creationDateSeconds, void *callbackData){
uint64_t * count = (uint64_t*) callbackData;
*count++;
*count += 1;
return S3StatusOK;
}
@ -381,11 +381,37 @@ static void S3_Delete(char *path, aiori_mod_opt_t * options)
}while(req.truncated);
S3_delete_bucket(o->s3_protocol, S3UriStylePath, o->access_key, o->secret_key, NULL, o->host, p, o->authRegion, NULL, o->timeout, & responseHandler, NULL);
}else{
s3_delete_req req = {0, o, 0, NULL};
do{
S3_list_bucket(& o->bucket_context, p, req.nextMarker, NULL, INT_MAX, NULL, o->timeout, & list_delete_handler, & req);
}while(req.truncated);
S3_delete_object(& o->bucket_context, p, NULL, o->timeout, & responseHandler, NULL);
char * del_heuristics = getenv("S3LIB_DELETE_HEURISTICS");
if(del_heuristics){
struct stat buf;
S3_head_object(& o->bucket_context, p, NULL, o->timeout, & statResponseHandler, & buf);
if(s3status != S3StatusOK){
// As the file does not exist, can return safely
CHECK_ERROR(p);
return;
}
int threshold = atoi(del_heuristics);
if (buf.st_size > threshold){
// there may exist fragments, so try to delete them
s3_delete_req req = {0, o, 0, NULL};
do{
S3_list_bucket(& o->bucket_context, p, req.nextMarker, NULL, INT_MAX, NULL, o->timeout, & list_delete_handler, & req);
}while(req.truncated);
}
S3_delete_object(& o->bucket_context, p, NULL, o->timeout, & responseHandler, NULL);
}else{
// Regular deletion, must remove all created fragments
S3_delete_object(& o->bucket_context, p, NULL, o->timeout, & responseHandler, NULL);
if(s3status != S3StatusOK){
// As the file does not exist, can return savely
CHECK_ERROR(p);
return;
}
s3_delete_req req = {0, o, 0, NULL};
do{
S3_list_bucket(& o->bucket_context, p, req.nextMarker, NULL, INT_MAX, NULL, o->timeout, & list_delete_handler, & req);
}while(req.truncated);
}
}
CHECK_ERROR(p);
}