vitalif
/
likeopera-backend
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move fetchFullMessage to Syncer

master
Vitaliy Filippov 2019-05-17 16:57:37 +03:00
parent 37bcd22719
commit 3b62de9e2d
7 changed files with 154 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ImapManager.js
View File

@ -152,7 +152,7 @@ class ImapManager
}
resolve();
})
.catch(e => reject(e));
.catch(reject);
}
};

47
Syncer.js
View File

@ -1,10 +1,11 @@
const Imap = require('imap');
const EventEmitter = require('events').EventEmitter;
const MailParser = require('mailparser').MailParser;
const iconv = require('iconv-lite');
const MailParser = require('mailparser').MailParser;
const mimelib = require('mimelib');
const ImapManager = require('./ImapManager.js');
const sanitizeHtml = require('./sanitize.js');
const SQL = require('./select-builder-pgsql.js');
class Syncer
@ -19,6 +20,7 @@ class Syncer
this.events = new EventEmitter();
}
// public
async init(cfg)
{
for (var i = 0; i < cfg.accounts.length; i++)
@ -28,6 +30,7 @@ class Syncer
await this.loadAccounts();
}
// public
async syncAll()
{
this.syncInProgress = true;
@ -148,6 +151,7 @@ class Syncer
this.imap.releaseConnection(accountId, 'S');
}
// public
async syncAccount(account)
{
let accountId = await SQL.select(this.pg, 'accounts', 'id', { email: account.email }, null, SQL.MS_VALUE);
@ -423,7 +427,7 @@ class Syncer
attachments.push([
mimelib.parseMimeWords(struct[i].disposition.params && struct[i].disposition.params.filename || struct[i].description || ''),
struct[i].type+'/'+struct[i].subtype,
struct[i].size
struct[i].size,
]);
}
}
@ -531,6 +535,37 @@ class Syncer
await SQL.update(this.pg, 'threads', upd, { id: msgrow.thread_id });
}
}
async fetchFullMessage(account_id, folder_id, folder_name, msg_uid)
{
// FIXME: parse and save attachments
// FIXME: replace inline images
let srv = await this.imap.getConnection(account_id, folder_name);
let upd = await this.imap.runFetch(
srv, msg_uid, { bodies: '' },
async(messages, state) => await this._parseBody(messages, folder_id)
);
this.imap.releaseConnection(account_id);
return upd;
}
async _parseBody(messages, boxId)
{
for (let i = 0; i < messages.length; i++)
{
let msg = messages[i];
let obj = await this.parseMsg(msg[0].headers);
obj.html = sanitizeHtml(obj.html);
let upd = { body_text: obj.text||'', body_html: obj.html };
upd.body_html_text = obj.html.replace(/<style[^>]*>.*<\/style\s*>|<\/?[^>]*>/g, '');
await SQL.update(this.pg, 'messages m', upd, { folder_id: boxId, uid: msg[0].uid });
if (messages.length == 1)
{
return [ upd ];
}
}
return null;
}
}
function toPgArray(a)
@ -539,4 +574,12 @@ function toPgArray(a)
return '{'+a.substring(1, a.length-1)+'}';
}
function mapToHash(map)
{
let h = {};
for (let v of map)
h[v[0]] = v[1];
return h;
}
module.exports = Syncer;

102
SyncerWeb.js
View File

@ -1,15 +1,12 @@
const MailParser = require('mailparser').MailParser;
const htmlawed = require('htmlawed');
const http = require('http');
const socket_io = require('socket.io');
const express = require('express');
const express_session = require('express-session');
const morgan = require('morgan');
const bodyparser = require('body-parser');
const multer = require('multer');
const css = require('css');
const SQL = require('./select-builder-pgsql.js');
const MAX_FETCH = 100;
@ -24,6 +21,8 @@ class SyncerWeb
this.app = express();
this.http = http.Server(this.app);
this.io = socket_io(this.http);
this.app.enable('trust proxy');
this.app.use(morgan('combined'));
this.app.use(bodyparser.urlencoded({ extended: false }));
this.app.use(express_session({
secret: this.cfg.sessionSecret || '1083581xm1l3s1l39k',
@ -238,7 +237,6 @@ class SyncerWeb
return res.sendStatus(401);
}
let msgId = req.query.msgId;
console.log('fetch message '+msgId);
let msg = await SQL.select(
this.pg,
{ m: 'messages', f: [ 'INNER', 'folders', [ 'f.id=m.folder_id' ] ] },
@ -252,12 +250,7 @@ class SyncerWeb
}
if (!msg.body_html && !msg.body_text)
{
let srv = await this.syncer.imap.getConnection(msg.account_id, msg.folder_name);
let upd = (await this.syncer.imap.runFetch(
srv, msg.uid, { bodies: '' },
async (messages, state) => await this.getBody(messages, msg.folder_id)
));
this.syncer.imap.releaseConnection(msg.account_id);
let upd = await this.syncer.fetchFullMessage(msg.account_id, msg.folder_id, msg.folder_name, msg.uid);
return res.send({ msg: { ...msg, ...upd[0] } });
}
return res.send({ msg: msg });
@ -281,89 +274,18 @@ class SyncerWeb
this.syncer.syncAll().catch(console.error);
return res.send({ status: 'started' });
}
async getBody(messages, boxId)
{
for (let i = 0; i < messages.length; i++)
{
let msg = messages[i];
let obj = await this.syncer.parseMsg(msg[0].headers);
obj.html = sanitizeHtml(obj.html);
let upd = { body_text: obj.text||'', body_html: obj.html };
upd.body_html_text = obj.html.replace(/<style[^>]*>.*<\/style\s*>|<\/?[^>]*>/g, '');
await SQL.update(this.pg, 'messages m', upd, { folder_id: boxId, uid: msg[0].uid });
if (messages.length == 1)
{
return [ upd ];
}
}
return null;
}
}
function rewriteCss(ast)
{
var rules = ast.rules || ast.stylesheet && ast.stylesheet.rules;
if (ast.stylesheet && ast.stylesheet.parsingErrors)
{
delete ast.stylesheet.parsingErrors;
}
if (rules)
{
for (var i = 0; i < rules.length; i++)
{
if (rules[i].type == 'document')
{
// prune @document instructions (may spy on current URL)
rules.splice(i--, 1);
}
else if (rules[i].type == 'rule' && (!rules[i].selectors || !rules[i].declarations))
rules.splice(i--, 1);
else
rewriteCss(rules[i]);
}
}
else if (ast.type == 'rule')
{
for (var i = 0; i < ast.selectors.length; i++)
{
// FIXME: Do not hardcode css selector for frontend here
// This will require generating unique substitution string,
// so we may also generate 'blocked images' stubs when we do it.
ast.selectors[i] = '.message-view .text '+ast.selectors[i];
}
}
}
function sanitizeHtml(html)
{
let styles = '';
html = (html||'').replace(/<style[^<>]*>([\s\S]*?)<\/style\s*>/ig, function(m, m1)
{
styles += m1+'\n';
return '';
});
html = html.replace(/^[\s\S]*?<body[^<>]*>([\s\S]*)<\/body>[\s\S]*$/i, '$1');
html = html.replace(/^[\s\S]*?<html[^<>]*>([\s\S]*)<\/html>[\s\S]*$/i, '$1');
if (styles)
{
html = '<style>\n'+styles+'</style>\n'+html;
styles = '';
}
html = htmlawed.sanitize(html||'', { safe: 1, elements: '* +style', keep_bad: 0, comment: 1 });
html = html.replace(/<a(\s*[^>]+)>/i, (m, m1) => '<a'+m1+' target="_blank">');
html = html.replace(/<style[^>]*>([\s\S]*)<\/style\s*>/ig, function(m, m1)
{
let ast = css.parse(m1, { silent: true });
rewriteCss(ast);
return '<style>'+css.stringify(ast)+'</style>';
});
return html;
}
function wrapAsync(self, fn)
{
return (req, res) => self[fn](req, res).catch(e => res.status(500).send('Internal Error: '+e.stack));
return (req, res) =>
{
self[fn](req, res).catch(e =>
{
console.error(e);
res.status(500).send('Internal Error: '+e.stack);
});
};
}
function ymd(dt)

22
database/db.sql
View File

@ -20,6 +20,26 @@ create table folders (
);
create unique index folders_name on folders (account_id, name);
-- Очередь команд. Виды команд:
-- 1) пометить сообщение
-- 2) переместить сообщение
-- 3) удалить сообщение
-- 4) отправить черновик
create table command_queue (
id serial not null primary key,
account_id int not null,
folder_id int,
uid int,
props jsonb not null
);
-- Ещё нигде не сохранённые черновики сообщений
create table drafts (
id serial not null primary key,
time bigint not null,
props jsonb not null
);
create table messages (
id serial not null primary key,
thread_id int,
@ -33,8 +53,10 @@ create table messages (
body_html text not null default '',
body_text text not null default '',
body_html_text text not null default '',
-- FIXME bigint
time timestamptz not null,
size int not null,
-- FIXME jsonb
flags text[] not null,
foreign key (folder_id) references folders (id) on delete cascade on update cascade
);

2
operetta.js
View File

@ -60,8 +60,8 @@ async function startSync(cfg)
let syncer = new Syncer(dbh);
let syncerweb = new SyncerWeb(syncer, dbh, cfg);
await syncer.init(cfg);
await syncer.syncAll();
syncerweb.listen(8057);
await syncer.syncAll();
}
let cfg = require('./cfg.json');

3
package.json
View File

@ -11,11 +11,12 @@
"css": "latest",
"express": "latest",
"express-session": "latest",
"htmlawed": "latest",
"htmlawed": "^1.0.2",
"iconv-lite": "latest",
"imap": "^0.8.19",
"mailparser": "git+https://github.com/vitalif/mailparser#master",
"mimelib": "git+https://github.com/vitalif/mimelib#master",
"morgan": "^1.9.1",
"multer": "latest",
"nodemailer": "latest",
"pg": "^7.10.0",

71
sanitize.js Normal file
View File

@ -0,0 +1,71 @@
const htmlawed = require('htmlawed');
const css = require('css');
function sanitizeHtml(html)
{
if (!html)
return '';
else
html = html+'';
let styles = '';
// GitHub tends to insert some metadata script. Cut off them here,
// because htmLawed has a global policy for bad tags and we use "leave content in place".
html = html.replace(/<script[^<>]*>([\s\S]*)<\/script\s*>/ig, '');
html = html.replace(/<style[^<>]*>([\s\S]*?)(<\/style\s*>|(?=<style[^<>]*>))/ig, function(m, m1)
{
styles += m1+'\n';
return '';
});
html = html.replace(/^[\s\S]*?<body[^<>]*>([\s\S]*)<\/body>[\s\S]*$/i, '$1');
html = html.replace(/^[\s\S]*?<html[^<>]*>([\s\S]*)<\/html>[\s\S]*$/i, '$1');
if (styles)
{
html = '<style>\n'+styles+'</style>\n'+html;
styles = '';
}
html = htmlawed.sanitize(html||'', { safe: 1, elements: '* +style', keep_bad: 6, comment: 1 });
html = html.replace(/<a(\s*[^>]+)>/ig, (m, m1) => '<a'+m1+' target="_blank">');
html = html.replace(/<style[^>]*>([\s\S]*)<\/style\s*>/ig, function(m, m1)
{
let ast = css.parse(m1, { silent: true });
rewriteCss(ast);
return '<style>'+css.stringify(ast)+'</style>';
});
return html;
}
function rewriteCss(ast)
{
var rules = ast.rules || ast.stylesheet && ast.stylesheet.rules;
if (ast.stylesheet && ast.stylesheet.parsingErrors)
{
delete ast.stylesheet.parsingErrors;
}
if (rules)
{
for (var i = 0; i < rules.length; i++)
{
if (rules[i].type == 'document')
{
// prune @document instructions (may spy on current URL)
rules.splice(i--, 1);
}
else if (rules[i].type == 'rule' && (!rules[i].selectors || !rules[i].declarations))
rules.splice(i--, 1);
else
rewriteCss(rules[i]);
}
}
else if (ast.type == 'rule')
{
for (var i = 0; i < ast.selectors.length; i++)
{
// FIXME: Do not hardcode css selector for frontend here
// This will require generating unique substitution string,
// so we may also generate 'blocked images' stubs when we do it.
ast.selectors[i] = '.message-view .text '+ast.selectors[i];
}
}
}
module.exports = sanitizeHtml;