guess encodings, sync all accounts

db.sql

@@ -21,7 +21,7 @@ create table folders (
     unread_count int not null,
     foreign key (account_id) references accounts (id) on delete cascade on update cascade
 );
-create unique index folders_name on folders (name);
+create unique index folders_name on folders (account_id, name);
 
 create table messages (
     id serial not null primary key,

operetta.js

@@ -1,3 +1,7 @@
+// TODO: Висеть в виде демона и сразу получать новые письма (IDLE)
+// TODO: Сделать веб-сервер для обновления view
+// TODO: Сделать подписки на новые сообщения по вебсокетам
+
 process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
 var cfg = require('./cfg.json');
 
@@ -5,6 +9,7 @@ require('heapdump');
 var gen = require('gen-thread');
 var Imap = require('imap');
 var inspect = require('util').inspect;
+var iconv = require('iconv-lite');
 
 var bricks = require('pg-bricks');
 var pg = bricks.configure('postgresql://'+cfg.pg.user+':'+cfg.pg.password+'@'+(cfg.pg.host||'')+':'+cfg.pg.port+'/'+cfg.pg.database);
@@ -39,7 +44,7 @@ Syncer.sync = function*(account)
         accountId = rows[0].id;
     else
     {
-        var [ row ] = pg.insert('accounts', {
+        var [ row ] = yield pg.insert('accounts', {
             name: account.name,
             email: account.email,
             settings: {
@@ -59,10 +64,11 @@ Syncer.sync = function*(account)
         var boxId;
         // IMAP sync: http://tools.ietf.org/html/rfc4549
         var [ row ] = yield pg.select('*').from('folders')
-            .where({ account_id: accountId, name: box.name }).row(gen.ef());
+            .where({ account_id: accountId, name: box.name }).rows(gen.ef());
         self.versionTag = 0;
-        if (row)
+        if (row.length)
         {
+            row = row[0];
             boxId = row.id;
             if (row.uidvalidity != box.uidvalidity)
             {
@@ -80,7 +86,7 @@ Syncer.sync = function*(account)
         }
         else
         {
-            var [ row ] = yield pg.insert('folders', {
+            [ row ] = yield pg.insert('folders', {
                 name: box.name,
                 uidvalidity: box.uidvalidity,
                 account_id: accountId,
@@ -90,6 +96,7 @@ Syncer.sync = function*(account)
             boxId = row.id;
         }
 
+        self.missing = [];
         var [ maxUid ] = yield pg.select('MAX(uid)').from('messages').where({ folder_id: boxId }).val(gen.ef());
         if (maxUid)
         {
@@ -125,7 +132,8 @@ Syncer.sync = function*(account)
         }
 
         // fetch new messages
-        yield* self.runFetch((maxUid ? maxUid+1 : 1)+':*', {
+        self.missing.push((maxUid ? maxUid+1 : 1)+':*');
+        yield* self.runFetch(self.missing, {
             size: true,
             bodies: 'HEADER'
         }, boxId, 'saveMessages');
@@ -221,15 +229,28 @@ Syncer.parseMessage = function*(msg, seqnum, boxId)
     var attrs;
     msg.on('body', function(stream, info)
     {
-        var buffer = '';
+        var buffer;
         stream.on('data', function(chunk)
         {
-            buffer += chunk.toString('utf8');
+            if (!buffer)
+                buffer = chunk;
+            else
+                buffer = Buffer.concat([ buffer, chunk ]);
         });
         stream.once('end', function()
         {
             msgrow.body = '';
-            msgrow.headers = buffer;
+            var b = buffer.toString('utf8');
+            if (b.indexOf('<27>') >= 0)
+            {
+                let enc = /Content-type:\s*[^;\n]*;\s*charset=(\S+)/i.exec(b);
+                enc = enc ? enc[1] : 'windows-1251';
+                try { b = iconv.decode(buffer, enc); }
+                catch (e) {}
+            }
+            if (b.indexOf('\0') >= 0)
+                b = b.substr(0, b.indexOf('\0'));
+            msgrow.headers = b;
         });
     });
     msg.once('attributes', function(a) {
@@ -243,7 +264,7 @@ Syncer.parseMessage = function*(msg, seqnum, boxId)
     msgrow.flags = nf.length == msgrow.flags.length ? nf.concat(['unread']) : nf;
     // Workaround memory leak in node-imap
     // TODO: send pull request
-    if (this.srv._curReq.fetchCache)
+    if (this.srv._curReq && this.srv._curReq.fetchCache)
         delete this.srv._curReq.fetchCache[seqnum];
     return [ msgrow, attrs ];
 }
@@ -264,7 +285,7 @@ Syncer.updateFlags = function*(messages, boxId)
         uh[updated[i].uid] = true;
     for (i = 0; i < messages.length; i++)
         if (!uh[messages[i][0].uid])
-            console.log('message is missing: '+messages[i][0].uid);
+            self.missing.push(messages[i][0].uid);
     self.synced += messages.length;
     process.stderr.write('\rsynchronizing '+self.synced);
 }
@@ -378,4 +399,9 @@ Syncer.addMessage = function*(msgrow, attrs)
     }
 }
 
-gen.run(Syncer.sync(cfg.accounts[0]), function() { process.exit() });
+gen.run(function*()
+{
+    for (var i = 0; i < cfg.accounts.length; i++)
+        yield* Syncer.sync(cfg.accounts[i]);
+    process.exit();
+});

package.json

@@ -8,6 +8,7 @@
   "description": "Operetta webmail backend",
   "dependencies": {
     "gen-thread": "latest",
+    "iconv-lite": "latest",
     "imap": "latest",
     "mailparser": "latest",
     "nodemailer": "latest",