guess encodings, sync all accounts
parent
7a25344a4a
commit
f4a4fcc1b2
2
db.sql
2
db.sql
|
@ -21,7 +21,7 @@ create table folders (
|
||||||
unread_count int not null,
|
unread_count int not null,
|
||||||
foreign key (account_id) references accounts (id) on delete cascade on update cascade
|
foreign key (account_id) references accounts (id) on delete cascade on update cascade
|
||||||
);
|
);
|
||||||
create unique index folders_name on folders (name);
|
create unique index folders_name on folders (account_id, name);
|
||||||
|
|
||||||
create table messages (
|
create table messages (
|
||||||
id serial not null primary key,
|
id serial not null primary key,
|
||||||
|
|
48
operetta.js
48
operetta.js
|
@ -1,3 +1,7 @@
|
||||||
|
// TODO: Висеть в виде демона и сразу получать новые письма (IDLE)
|
||||||
|
// TODO: Сделать веб-сервер для обновления view
|
||||||
|
// TODO: Сделать подписки на новые сообщения по вебсокетам
|
||||||
|
|
||||||
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
||||||
var cfg = require('./cfg.json');
|
var cfg = require('./cfg.json');
|
||||||
|
|
||||||
|
@ -5,6 +9,7 @@ require('heapdump');
|
||||||
var gen = require('gen-thread');
|
var gen = require('gen-thread');
|
||||||
var Imap = require('imap');
|
var Imap = require('imap');
|
||||||
var inspect = require('util').inspect;
|
var inspect = require('util').inspect;
|
||||||
|
var iconv = require('iconv-lite');
|
||||||
|
|
||||||
var bricks = require('pg-bricks');
|
var bricks = require('pg-bricks');
|
||||||
var pg = bricks.configure('postgresql://'+cfg.pg.user+':'+cfg.pg.password+'@'+(cfg.pg.host||'')+':'+cfg.pg.port+'/'+cfg.pg.database);
|
var pg = bricks.configure('postgresql://'+cfg.pg.user+':'+cfg.pg.password+'@'+(cfg.pg.host||'')+':'+cfg.pg.port+'/'+cfg.pg.database);
|
||||||
|
@ -39,7 +44,7 @@ Syncer.sync = function*(account)
|
||||||
accountId = rows[0].id;
|
accountId = rows[0].id;
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
var [ row ] = pg.insert('accounts', {
|
var [ row ] = yield pg.insert('accounts', {
|
||||||
name: account.name,
|
name: account.name,
|
||||||
email: account.email,
|
email: account.email,
|
||||||
settings: {
|
settings: {
|
||||||
|
@ -59,10 +64,11 @@ Syncer.sync = function*(account)
|
||||||
var boxId;
|
var boxId;
|
||||||
// IMAP sync: http://tools.ietf.org/html/rfc4549
|
// IMAP sync: http://tools.ietf.org/html/rfc4549
|
||||||
var [ row ] = yield pg.select('*').from('folders')
|
var [ row ] = yield pg.select('*').from('folders')
|
||||||
.where({ account_id: accountId, name: box.name }).row(gen.ef());
|
.where({ account_id: accountId, name: box.name }).rows(gen.ef());
|
||||||
self.versionTag = 0;
|
self.versionTag = 0;
|
||||||
if (row)
|
if (row.length)
|
||||||
{
|
{
|
||||||
|
row = row[0];
|
||||||
boxId = row.id;
|
boxId = row.id;
|
||||||
if (row.uidvalidity != box.uidvalidity)
|
if (row.uidvalidity != box.uidvalidity)
|
||||||
{
|
{
|
||||||
|
@ -80,7 +86,7 @@ Syncer.sync = function*(account)
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
var [ row ] = yield pg.insert('folders', {
|
[ row ] = yield pg.insert('folders', {
|
||||||
name: box.name,
|
name: box.name,
|
||||||
uidvalidity: box.uidvalidity,
|
uidvalidity: box.uidvalidity,
|
||||||
account_id: accountId,
|
account_id: accountId,
|
||||||
|
@ -90,6 +96,7 @@ Syncer.sync = function*(account)
|
||||||
boxId = row.id;
|
boxId = row.id;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
self.missing = [];
|
||||||
var [ maxUid ] = yield pg.select('MAX(uid)').from('messages').where({ folder_id: boxId }).val(gen.ef());
|
var [ maxUid ] = yield pg.select('MAX(uid)').from('messages').where({ folder_id: boxId }).val(gen.ef());
|
||||||
if (maxUid)
|
if (maxUid)
|
||||||
{
|
{
|
||||||
|
@ -125,7 +132,8 @@ Syncer.sync = function*(account)
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetch new messages
|
// fetch new messages
|
||||||
yield* self.runFetch((maxUid ? maxUid+1 : 1)+':*', {
|
self.missing.push((maxUid ? maxUid+1 : 1)+':*');
|
||||||
|
yield* self.runFetch(self.missing, {
|
||||||
size: true,
|
size: true,
|
||||||
bodies: 'HEADER'
|
bodies: 'HEADER'
|
||||||
}, boxId, 'saveMessages');
|
}, boxId, 'saveMessages');
|
||||||
|
@ -221,15 +229,28 @@ Syncer.parseMessage = function*(msg, seqnum, boxId)
|
||||||
var attrs;
|
var attrs;
|
||||||
msg.on('body', function(stream, info)
|
msg.on('body', function(stream, info)
|
||||||
{
|
{
|
||||||
var buffer = '';
|
var buffer;
|
||||||
stream.on('data', function(chunk)
|
stream.on('data', function(chunk)
|
||||||
{
|
{
|
||||||
buffer += chunk.toString('utf8');
|
if (!buffer)
|
||||||
|
buffer = chunk;
|
||||||
|
else
|
||||||
|
buffer = Buffer.concat([ buffer, chunk ]);
|
||||||
});
|
});
|
||||||
stream.once('end', function()
|
stream.once('end', function()
|
||||||
{
|
{
|
||||||
msgrow.body = '';
|
msgrow.body = '';
|
||||||
msgrow.headers = buffer;
|
var b = buffer.toString('utf8');
|
||||||
|
if (b.indexOf('<27>') >= 0)
|
||||||
|
{
|
||||||
|
let enc = /Content-type:\s*[^;\n]*;\s*charset=(\S+)/i.exec(b);
|
||||||
|
enc = enc ? enc[1] : 'windows-1251';
|
||||||
|
try { b = iconv.decode(buffer, enc); }
|
||||||
|
catch (e) {}
|
||||||
|
}
|
||||||
|
if (b.indexOf('\0') >= 0)
|
||||||
|
b = b.substr(0, b.indexOf('\0'));
|
||||||
|
msgrow.headers = b;
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
msg.once('attributes', function(a) {
|
msg.once('attributes', function(a) {
|
||||||
|
@ -243,7 +264,7 @@ Syncer.parseMessage = function*(msg, seqnum, boxId)
|
||||||
msgrow.flags = nf.length == msgrow.flags.length ? nf.concat(['unread']) : nf;
|
msgrow.flags = nf.length == msgrow.flags.length ? nf.concat(['unread']) : nf;
|
||||||
// Workaround memory leak in node-imap
|
// Workaround memory leak in node-imap
|
||||||
// TODO: send pull request
|
// TODO: send pull request
|
||||||
if (this.srv._curReq.fetchCache)
|
if (this.srv._curReq && this.srv._curReq.fetchCache)
|
||||||
delete this.srv._curReq.fetchCache[seqnum];
|
delete this.srv._curReq.fetchCache[seqnum];
|
||||||
return [ msgrow, attrs ];
|
return [ msgrow, attrs ];
|
||||||
}
|
}
|
||||||
|
@ -264,7 +285,7 @@ Syncer.updateFlags = function*(messages, boxId)
|
||||||
uh[updated[i].uid] = true;
|
uh[updated[i].uid] = true;
|
||||||
for (i = 0; i < messages.length; i++)
|
for (i = 0; i < messages.length; i++)
|
||||||
if (!uh[messages[i][0].uid])
|
if (!uh[messages[i][0].uid])
|
||||||
console.log('message is missing: '+messages[i][0].uid);
|
self.missing.push(messages[i][0].uid);
|
||||||
self.synced += messages.length;
|
self.synced += messages.length;
|
||||||
process.stderr.write('\rsynchronizing '+self.synced);
|
process.stderr.write('\rsynchronizing '+self.synced);
|
||||||
}
|
}
|
||||||
|
@ -378,4 +399,9 @@ Syncer.addMessage = function*(msgrow, attrs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
gen.run(Syncer.sync(cfg.accounts[0]), function() { process.exit() });
|
gen.run(function*()
|
||||||
|
{
|
||||||
|
for (var i = 0; i < cfg.accounts.length; i++)
|
||||||
|
yield* Syncer.sync(cfg.accounts[i]);
|
||||||
|
process.exit();
|
||||||
|
});
|
||||||
|
|
|
@ -8,6 +8,7 @@
|
||||||
"description": "Operetta webmail backend",
|
"description": "Operetta webmail backend",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"gen-thread": "latest",
|
"gen-thread": "latest",
|
||||||
|
"iconv-lite": "latest",
|
||||||
"imap": "latest",
|
"imap": "latest",
|
||||||
"mailparser": "latest",
|
"mailparser": "latest",
|
||||||
"nodemailer": "latest",
|
"nodemailer": "latest",
|
||||||
|
|
Loading…
Reference in New Issue