vitalif
/
mdtest
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

S3-libs3 error handling improved, added a heuristic for testing as well.

master
Julian M. Kunkel 2021-04-23 12:20:11 +02:00
parent a90d414a30
commit be80b4e618
1 changed files with 32 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/aiori-S3-libs3.c
View File

@ -171,7 +171,7 @@ static void S3_Sync(aiori_mod_opt_t * options)
static S3Status S3ListResponseCallback(const char *ownerId, const char *ownerDisplayName, const char *bucketName, int64_t creationDateSeconds, void *callbackData){
uint64_t * count = (uint64_t*) callbackData;
*count++;
*count += 1;
return S3StatusOK;
}
@ -381,11 +381,37 @@ static void S3_Delete(char *path, aiori_mod_opt_t * options)
}while(req.truncated);
S3_delete_bucket(o->s3_protocol, S3UriStylePath, o->access_key, o->secret_key, NULL, o->host, p, o->authRegion, NULL, o->timeout, & responseHandler, NULL);
}else{
s3_delete_req req = {0, o, 0, NULL};
do{
S3_list_bucket(& o->bucket_context, p, req.nextMarker, NULL, INT_MAX, NULL, o->timeout, & list_delete_handler, & req);
}while(req.truncated);
S3_delete_object(& o->bucket_context, p, NULL, o->timeout, & responseHandler, NULL);
char * del_heuristics = getenv("S3LIB_DELETE_HEURISTICS");
if(del_heuristics){
struct stat buf;
S3_head_object(& o->bucket_context, p, NULL, o->timeout, & statResponseHandler, & buf);
if(s3status != S3StatusOK){
// As the file does not exist, can return safely
CHECK_ERROR(p);
return;
}
int threshold = atoi(del_heuristics);
if (buf.st_size > threshold){
// there may exist fragments, so try to delete them
s3_delete_req req = {0, o, 0, NULL};
do{
S3_list_bucket(& o->bucket_context, p, req.nextMarker, NULL, INT_MAX, NULL, o->timeout, & list_delete_handler, & req);
}while(req.truncated);
}
S3_delete_object(& o->bucket_context, p, NULL, o->timeout, & responseHandler, NULL);
}else{
// Regular deletion, must remove all created fragments
S3_delete_object(& o->bucket_context, p, NULL, o->timeout, & responseHandler, NULL);
if(s3status != S3StatusOK){
// As the file does not exist, can return savely
CHECK_ERROR(p);
return;
}
s3_delete_req req = {0, o, 0, NULL};
do{
S3_list_bucket(& o->bucket_context, p, req.nextMarker, NULL, INT_MAX, NULL, o->timeout, & list_delete_handler, & req);
}while(req.truncated);
}
}
CHECK_ERROR(p);
}