224 lines
9.1 KiB
JavaScript
224 lines
9.1 KiB
JavaScript
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
|
var cfg = require('./cfg.json');
|
|
|
|
require('heapdump');
|
|
var gen = require('gen-thread');
|
|
var Imap = require('imap');
|
|
var inspect = require('util').inspect;
|
|
//var pg;
|
|
//try { require('pg-native'); pg = require('pg').native; }
|
|
//catch(e) { pg = require('pg'); }
|
|
//var pg_pool = new pg.Pool(cfg.pg);
|
|
|
|
var bricks = require('pg-bricks');
|
|
var pg = bricks.configure('postgresql://'+cfg.pg.user+':'+cfg.pg.password+'@'+(cfg.pg.host||'')+':'+cfg.pg.port+'/'+cfg.pg.database);
|
|
|
|
var flagNum = {
|
|
'\\recent': 1,
|
|
'\\flagged': 2,
|
|
'\\answered': 4,
|
|
'\\seen': 8,
|
|
};
|
|
|
|
function splitEmails(s)
|
|
{
|
|
var re = /^[\s,]*(?:(?:["'](.*?)["']|([^<]+))\s*<([^>]+)>|<?([^<>]+)>?)/; // '
|
|
var m, r = [];
|
|
while (m = re.exec(s))
|
|
{
|
|
s = s.substr(m[0].length);
|
|
r.push({ name: (m[1]||m[2]||'').trim(), email: (m[3]||m[4]||'').trim() });
|
|
}
|
|
return r;
|
|
}
|
|
|
|
function toPgArray(a)
|
|
{
|
|
a = JSON.stringify(a);
|
|
return '{'+a.substring(1, a.length-1)+'}';
|
|
}
|
|
|
|
function* main(NEXT, account)
|
|
{
|
|
var accountId;
|
|
var [ err, rows ] = yield pg.select('id').from('accounts').where({ email: account.email }).rows(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
if (rows[0] && rows[0].id)
|
|
accountId = rows[0].id;
|
|
else
|
|
{
|
|
var [ err, row ] = yield pg.insert('accounts', {
|
|
name: account.name,
|
|
email: account.email,
|
|
settings: {
|
|
imap: account.imap
|
|
}
|
|
}).returning('id').row(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
accountId = row.id;
|
|
}
|
|
var srv = new Imap(account.imap);
|
|
srv.once('ready', NEXT.cb());
|
|
yield srv.connect();
|
|
var [ err, boxes ] = yield srv.getBoxes(NEXT.cb());
|
|
for (var k in boxes)
|
|
{
|
|
var [ err, box ] = yield srv.openBox(k, true, NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
var boxId;
|
|
var [ err, rows ] = yield pg.update('folders', { uidvalidity: box.uidvalidity, unread_count: box.messages.new })
|
|
.where({ account_id: accountId, name: box.name }).returning('id').rows(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
if (rows[0] && rows[0].id)
|
|
{
|
|
// IMAP sync: http://tools.ietf.org/html/rfc4549
|
|
// TODO: check old uidvalidity
|
|
boxId = rows[0].id;
|
|
}
|
|
else
|
|
{
|
|
var [ err, row ] = yield pg.insert('folders', {
|
|
name: box.name,
|
|
uidvalidity: box.uidvalidity,
|
|
account_id: accountId,
|
|
unread_count: box.messages.new,
|
|
// total_count: box.messages.count
|
|
}).returning('id').row(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
boxId = row.id;
|
|
}
|
|
var f = srv.fetch('1:*', {
|
|
size: true,
|
|
bodies: 'HEADER'
|
|
});
|
|
f.on('message', function(msg, seqnum)
|
|
{
|
|
gen.run(function*(NEXT)
|
|
{
|
|
var msgrow = {};
|
|
var attrs;
|
|
msg.on('body', function(stream, info)
|
|
{
|
|
var buffer = '';
|
|
stream.on('data', function(chunk)
|
|
{
|
|
buffer += chunk.toString('utf8');
|
|
});
|
|
stream.once('end', function()
|
|
{
|
|
msgrow.body = '';
|
|
msgrow.headers = buffer;
|
|
});
|
|
});
|
|
msg.once('attributes', function(a) {
|
|
attrs = a;
|
|
});
|
|
yield msg.once('end', NEXT.cb());
|
|
|
|
// Workaround memory leak in node-imap
|
|
// TODO: send pull request
|
|
if (srv._curReq.fetchCache)
|
|
delete srv._curReq.fetchCache[seqnum];
|
|
/*var cached = 0;
|
|
for (var i in srv._curReq.fetchCache)
|
|
cached++;
|
|
console.log('msg '+seqnum+': '+cached+' in cache');*/
|
|
|
|
var [ pgtx, end_transaction ] = yield pg.transaction(NEXT.cb(), function(e) { if (e) throw e; });
|
|
//yield NEXT.throttle(20);
|
|
|
|
var header = Imap.parseHeader(msgrow.headers);
|
|
for (var i in header)
|
|
for (var k = 0; k < header[i].length; k++)
|
|
header[i][k] = header[i][k].replace(/\x00/g, '');
|
|
header.from = header.from && splitEmails(header.from[0])[0];
|
|
msgrow.from_email = header.from && header.from.email || '';
|
|
msgrow.from_name = header.from && header.from.name || '';
|
|
header.replyto = header['reply-to'] && splitEmails(header['reply-to'][0])[0];
|
|
msgrow.replyto_email = header.replyto && header.replyto.email || '';
|
|
msgrow.replyto_name = header.replyto && header.replyto.name || '';
|
|
msgrow.to_list = header.to && header.to[0] || '';
|
|
msgrow.cc_list = header.cc && header.cc[0] || '';
|
|
msgrow.bcc_list = header.bcc && header.bcc[0] || '';
|
|
msgrow.subject = header.subject && header.subject[0] || '';
|
|
msgrow.messageid = header['message-id'] && header['message-id'][0] || '';
|
|
msgrow.inreplyto = header['in-reply-to'] && header['in-reply-to'][0] || '';
|
|
msgrow.inreplyto = msgrow.inreplyto.replace(/^[\s\S]*(<[^>]*>)[\s\S]*$/, '$1');
|
|
var re = /(<[^>]*>)/;
|
|
var references = (header.references && header.references[0] || '').split(re).filter(a => a.match(re));
|
|
msgrow.refs = toPgArray(references);
|
|
if (references.length)
|
|
{
|
|
if (references.length > 10)
|
|
references = [ references[0] ].concat(references.slice(references.count-9));
|
|
if (!msgrow.inreplyto)
|
|
msgrow.inreplyto = references[references.length-1];
|
|
else if (references[references.length-1] != msgrow.inreplyto)
|
|
references.push(msgrow.inreplyto);
|
|
var [ err, threadId ] = yield pgtx.select('MAX(thread_id)').from('messages')
|
|
.where(pg.sql.in('messageid', references)).val(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
if (!threadId)
|
|
{
|
|
var [ err, threadId ] = yield pgtx.select('MAX(thread_id)').from('messages')
|
|
.where(new pg.sql.Binary('@>', 'refs', toPgArray([msgrow.messageid]))).val(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
}
|
|
if (threadId)
|
|
{
|
|
try
|
|
{
|
|
var [ err ] = yield pgtx.update('threads', { msg_count: pg.sql('msg_count+1') })
|
|
.where({ id: threadId }).run(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
}
|
|
catch (e)
|
|
{
|
|
throw new Error(''+e);
|
|
}
|
|
}
|
|
msgrow.thread_id = threadId;
|
|
}
|
|
if (header.date)
|
|
{
|
|
var t = Date.parse(header.date[0]);
|
|
if (!isNaN(t))
|
|
msgrow.time = new Date(t);
|
|
}
|
|
if (!msgrow.time)
|
|
msgrow.time = new Date(attrs.date);
|
|
msgrow.uid = attrs.uid;
|
|
msgrow.folder_id = boxId;
|
|
msgrow.flags = 0;
|
|
for (var i = 0; i < attrs.flags.length; i++)
|
|
msgrow.flags = msgrow.flags || flagNum[attrs.flags[i].toLowerCase()];
|
|
msgrow.flags = (msgrow.flags & ~8) | (msgrow.flags & 8 ? 0 : 8); // invert "\seen" (unread) flag
|
|
console.log(msgrow.time+' '+msgrow.from_email+' '+msgrow.subject);
|
|
var [ err, id ] = yield pgtx.raw(
|
|
pgtx.insert('messages', msgrow)+' ON CONFLICT (folder_id, uid) DO UPDATE SET flags=excluded.flags RETURNING id'
|
|
).val(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
msgrow.id = id;
|
|
if (!msgrow.thread_id)
|
|
{
|
|
var [ err, thread_id ] = yield pgtx.insert('threads', {
|
|
first_msg: msgrow.id,
|
|
msg_count: 1
|
|
}).returning('id').val(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
msgrow.thread_id = thread_id;
|
|
var [ err, row ] = yield pgtx.update('messages', { thread_id: msgrow.thread_id }).where({ id: msgrow.id }).run(NEXT.cb());
|
|
if (err) throw new Error(''+err);
|
|
}
|
|
|
|
end_transaction();
|
|
});
|
|
});
|
|
yield f.once('end', NEXT.cb());
|
|
yield srv.closeBox(NEXT.cb());
|
|
}
|
|
srv.end();
|
|
}
|
|
|
|
gen.run(main, cfg.accounts[0], function() { process.exit() });
|