Compare commits

..

2 Commits

Author SHA1 Message Date
Vitaliy Filippov fdfd50c981 Use TS? 2024-08-05 02:26:18 +03:00
Vitaliy Filippov 329d8ef32c Add Vitastor support 2024-08-05 02:23:54 +03:00
2 changed files with 74 additions and 19 deletions

View File

@ -10,8 +10,63 @@ const VOLUME_MAGIC = 'VstS3Vol';
const OBJECT_MAGIC = 'VstS3Obj'; const OBJECT_MAGIC = 'VstS3Obj';
const FLAG_DELETED = 2n; const FLAG_DELETED = 2n;
type Volume = {
id: number,
partial_sectors: {
[key: string]: {
buffer: Buffer,
refs: number,
},
},
header: {
location: string,
bucket: string,
max_size: number,
create_ts: number,
used_ts: number,
size: number,
objects: number,
removed_objects: number,
object_bytes: number,
removed_bytes: number,
},
};
type ObjectHeader = {
size: number,
key: string,
part_num?: number,
};
class VitastorBackend class VitastorBackend
{ {
locationName: string;
config: {
pack_objects: boolean,
[key: string]: any,
};
next_id: number;
alloc_id: number;
opened: boolean;
on_open: ((...args: any[]) => void)[] | null;
open_error: Error | null;
cli: any;
kv: any;
volumes: {
[bucket: string]: {
[max_size: string]: Volume,
},
};
volumes_by_id: {
[id: string]: Volume,
};
volume_delete_stats: {
[id: string]: {
count: number,
bytes: number,
},
};
constructor(locationName, config) constructor(locationName, config)
{ {
this.locationName = locationName; this.locationName = locationName;
@ -38,7 +93,7 @@ class VitastorBackend
this.config.pack_objects = !!this.config.pack_objects; this.config.pack_objects = !!this.config.pack_objects;
// state // state
this.next_id = 1; this.next_id = 1;
this.alloc_max_id = 0; this.alloc_id = 0;
this.opened = false; this.opened = false;
this.on_open = null; this.on_open = null;
this.open_error = null; this.open_error = null;
@ -57,7 +112,7 @@ class VitastorBackend
return this.next_id++; return this.next_id++;
} }
const id_key = 'id'+this.config.pool_id; const id_key = 'id'+this.config.pool_id;
const [ err, prev ] = await new Promise(ok => this.kv.get(id_key, (err, value) => ok([ err, value ]))); const [ err, prev ] = await new Promise<[ any, string ]>(ok => this.kv.get(id_key, (err, value) => ok([ err, value ])));
if (err && err != vitastor.ENOENT) if (err && err != vitastor.ENOENT)
{ {
throw new Error(err); throw new Error(err);
@ -65,7 +120,7 @@ class VitastorBackend
const new_id = (parseInt(prev) || 0) + 1; const new_id = (parseInt(prev) || 0) + 1;
this.next_id = new_id; this.next_id = new_id;
this.alloc_id = this.next_id + this.config.id_batch_size - 1; this.alloc_id = this.next_id + this.config.id_batch_size - 1;
await new Promise((ok, no) => this.kv.set(id_key, this.alloc_id, err => (err ? no(new Error(err)) : ok()), cas_old => cas_old === prev)); await new Promise((ok, no) => this.kv.set(id_key, this.alloc_id, err => (err ? no(new Error(err)) : ok(null)), cas_old => cas_old === prev));
return this.next_id; return this.next_id;
} }
@ -75,7 +130,7 @@ class VitastorBackend
{ {
if (this.on_open) if (this.on_open)
{ {
await new Promise(ok => this.on_open.push(ok)); await new Promise(ok => this.on_open!.push(ok));
} }
else else
{ {
@ -83,7 +138,7 @@ class VitastorBackend
if (this.config.metadata_image) if (this.config.metadata_image)
{ {
const img = new vitastor.Image(this.cli, this.config.metadata_image); const img = new vitastor.Image(this.cli, this.config.metadata_image);
const info = await new Promise(ok => img.get_info(ok)); const info = await new Promise<{ pool_id: number, inode_num: number }>(ok => img.get_info(ok));
this.config.metadata_pool_id = info.pool_id; this.config.metadata_pool_id = info.pool_id;
this.config.metadata_inode_num = info.inode_num; this.config.metadata_inode_num = info.inode_num;
} }
@ -147,10 +202,10 @@ class VitastorBackend
const buf = Buffer.alloc(this.config.sector_size); const buf = Buffer.alloc(this.config.sector_size);
buf.write(VOLUME_MAGIC + header_text, 0); buf.write(VOLUME_MAGIC + header_text, 0);
await new Promise((ok, no) => this.cli.write( await new Promise((ok, no) => this.cli.write(
this.config.pool_id, new_id, 0, buf, err => (err ? no(new Error(err)) : ok()) this.config.pool_id, new_id, 0, buf, err => (err ? no(new Error(err)) : ok(null))
)); ));
await new Promise((ok, no) => this.kv.set( await new Promise((ok, no) => this.kv.set(
'vol_'+this.config.pool_id+'_'+new_id, header_text, err => (err ? no(new Error(err)) : ok()), cas_old => !cas_old 'vol_'+this.config.pool_id+'_'+new_id, header_text, err => (err ? no(new Error(err)) : ok(null)), cas_old => !cas_old
)); ));
return new_vol; return new_vol;
} }
@ -253,7 +308,7 @@ class VitastorBackend
i++; i++;
} }
} }
if (last) if (write_all)
{ {
write_chunks.push(last_sect); write_chunks.push(last_sect);
write_size += this.config.sector_size; write_size += this.config.sector_size;
@ -296,7 +351,7 @@ class VitastorBackend
_put(vol, stream, size, keyContext, reqUids, callback) _put(vol, stream, size, keyContext, reqUids, callback)
{ {
const object_header = { const object_header: ObjectHeader = {
size, size,
key: keyContext.objectKey, key: keyContext.objectKey,
}; };
@ -315,7 +370,7 @@ class VitastorBackend
let cur_pos = object_pos; let cur_pos = object_pos;
let cur_chunks = [ object_header_buf ]; let cur_chunks = [ object_header_buf ];
let cur_size = object_header_buf.length; let cur_size = object_header_buf.length;
let err = null; let err: Error|null = null;
let waiting = 1; // 1 for end or error, 1 for each write request let waiting = 1; // 1 for end or error, 1 for each write request
vol.header.size += object_header_buf.length + size; vol.header.size += object_header_buf.length + size;
if (!this.config.pack_objects && (vol.header.size % this.config.sector_size)) if (!this.config.pack_objects && (vol.header.size % this.config.sector_size))
@ -418,7 +473,7 @@ class VitastorBackend
!(objectGetInfo.key instanceof Object) || !objectGetInfo.key.volume || !(objectGetInfo.key instanceof Object) || !objectGetInfo.key.volume ||
!objectGetInfo.key.offset || !objectGetInfo.key.hdrlen || !objectGetInfo.key.size) !objectGetInfo.key.offset || !objectGetInfo.key.hdrlen || !objectGetInfo.key.size)
{ {
throw new Error('objectGetInfo must be { key: { volume, offset, hdrlen, size } }, but is '+JSON.stringify(objectGetInfo, 0, 2)); throw new Error('objectGetInfo must be { key: { volume, offset, hdrlen, size } }, but is '+JSON.stringify(objectGetInfo));
} }
const [ start, end ] = range || []; const [ start, end ] = range || [];
if (start < 0 || end < 0 || end != null && start != null && end < start || start >= objectGetInfo.key.size) if (start < 0 || end < 0 || end != null && start != null && end < start || start >= objectGetInfo.key.size)
@ -451,7 +506,7 @@ class VitastorBackend
delete(objectGetInfo, reqUids, callback) delete(objectGetInfo, reqUids, callback)
{ {
callback = once(callback); callback = once(callback);
this._delete(objectGetInfo, reqUids, callback) this._delete(objectGetInfo, reqUids)
.then(callback) .then(callback)
.catch(callback); .catch(callback);
} }
@ -462,7 +517,7 @@ class VitastorBackend
!(objectGetInfo.key instanceof Object) || !objectGetInfo.key.volume || !(objectGetInfo.key instanceof Object) || !objectGetInfo.key.volume ||
!objectGetInfo.key.offset || !objectGetInfo.key.hdrlen || !objectGetInfo.key.size) !objectGetInfo.key.offset || !objectGetInfo.key.hdrlen || !objectGetInfo.key.size)
{ {
throw new Error('objectGetInfo must be { key: { volume, offset, hdrlen, size } }, but is '+JSON.stringify(objectGetInfo, 0, 2)); throw new Error('objectGetInfo must be { key: { volume, offset, hdrlen, size } }, but is '+JSON.stringify(objectGetInfo));
} }
const in_sect_pos = (objectGetInfo.key.offset % this.config.sector_size); const in_sect_pos = (objectGetInfo.key.offset % this.config.sector_size);
const sect_pos = objectGetInfo.key.offset - in_sect_pos; const sect_pos = objectGetInfo.key.offset - in_sect_pos;
@ -479,7 +534,7 @@ class VitastorBackend
del_stat.bytes += objectGetInfo.key.size; del_stat.bytes += objectGetInfo.key.size;
sect.buffer.writeBigInt64LE(flags | FLAG_DELETED, in_sect_pos + 8); sect.buffer.writeBigInt64LE(flags | FLAG_DELETED, in_sect_pos + 8);
sect.refs++; sect.refs++;
const err = await new Promise(ok => this.cli.write(this.config.pool_id, objectGetInfo.key.volume, sect_pos, sect.buffer, ok)); const err = await new Promise<any>(ok => this.cli.write(this.config.pool_id, objectGetInfo.key.volume, sect_pos, sect.buffer, ok));
sect.refs--; sect.refs--;
if (err) if (err)
{ {
@ -491,7 +546,7 @@ class VitastorBackend
else else
{ {
// RMW with CAS // RMW with CAS
const [ err, buf, version ] = await new Promise(ok => this.cli.read( const [ err, buf, version ] = await new Promise<[ any, Buffer, bigint ]>(ok => this.cli.read(
this.config.pool_id, objectGetInfo.key.volume, sect_pos, this.config.sector_size, this.config.pool_id, objectGetInfo.key.volume, sect_pos, this.config.sector_size,
(err, buf, version) => ok([ err, buf, version ]) (err, buf, version) => ok([ err, buf, version ])
)); ));
@ -508,7 +563,7 @@ class VitastorBackend
{ {
try try
{ {
json_hdr = JSON.parse(buf.slice(in_sect_pos+24, in_sect_pos+24+json_len)); json_hdr = JSON.parse(buf.slice(in_sect_pos+24, in_sect_pos+24+json_len).toString());
} }
catch (e) catch (e)
{ {
@ -524,7 +579,7 @@ class VitastorBackend
else if (!(flags & FLAG_DELETED)) else if (!(flags & FLAG_DELETED))
{ {
buf.writeBigInt64LE(flags | FLAG_DELETED, in_sect_pos + 8); buf.writeBigInt64LE(flags | FLAG_DELETED, in_sect_pos + 8);
const err = await new Promise(ok => this.cli.write(this.config.pool_id, objectGetInfo.key.volume, sect_pos, buf, { version: version+1n }, ok)); const err = await new Promise<any>(ok => this.cli.write(this.config.pool_id, objectGetInfo.key.volume, sect_pos, buf, { version: version+1n }, ok));
if (err == vitastor.EINTR) if (err == vitastor.EINTR)
{ {
// Retry // Retry
@ -557,7 +612,7 @@ class VitastorBackend
class VitastorReadStream extends stream.Readable class VitastorReadStream extends stream.Readable
{ {
constructor(cli, volume_id, offset, len, config, options) constructor(cli, volume_id, offset, len, config, options = undefined)
{ {
super(options); super(options);
this.cli = cli; this.cli = cli;

View File

@ -1,6 +1,6 @@
{ {
"compilerOptions": { "compilerOptions": {
"target": "es6", "target": "es2020",
"module": "commonjs", "module": "commonjs",
"rootDir": "./", "rootDir": "./",
"resolveJsonModule": true, "resolveJsonModule": true,