likeopera-backend/SyncerWeb.js

const MailParser = require('mailparser').MailParser;
const htmlawed = require('htmlawed');

const http = require('http');
const socket_io = require('socket.io');
const express = require('express');
const express_session = require('express-session');
const bodyparser = require('body-parser');
const multer = require('multer');

const css = require('css');

const SQL = require('./select-builder-pgsql.js');

const MAX_FETCH = 100;

class SyncerWeb
{
    constructor(syncer, pg, cfg)
    {
        this.syncer = syncer;
        this.pg = pg;
        this.cfg = cfg;
        this.app = express();
        this.http = http.Server(this.app);
        this.io = socket_io(this.http);
        this.app.use(bodyparser.urlencoded({ extended: false }));
        this.app.use(express_session({
            secret: this.cfg.sessionSecret || '1083581xm1l3s1l39k',
            resave: false,
            saveUninitialized: false
        }));
        this.app.get('/auth', (req, res) => this.get_auth(req, res));
        this.app.post('/auth', (req, res) => this.post_auth(req, res));
        this.app.get('/folders', wrapAsync(this, 'get_folders'));
        this.app.get('/groups', wrapAsync(this, 'get_groups'));
        this.app.get('/messages', wrapAsync(this, 'get_messages'));
        this.app.get('/message', wrapAsync(this, 'get_message'));
        this.app.post('/sync', wrapAsync(this, 'post_sync'));
        this.syncer.events.on('sync', (params) => this.syncer_sync(params));
    }

    listen(port)
    {
        this.http.listen(port);
    }

    get_auth(req, res)
    {
        return res.type('html').send(
            '<form action="/auth" method="post"><input name="login" />'+
            ' <input name="password" type="password" /> <input type="submit" /></form>'
        );
    }

    post_auth(req, res)
    {
        if (!req.body)
            return res.sendStatus(400);
        if (req.body.login == this.cfg.login && req.body.password == this.cfg.password)
        {
            req.session.auth = true;
            return res.send({ ok: true });
        }
        return res.send({ ok: false });
    }

    async get_folders(req, res)
    {
        if (this.cfg.login && (!req.session || !req.session.auth))
        {
            return res.sendStatus(401);
        }
        const accounts = (await this.pg.query(
            'select id, name, email, settings->\'folders\' folderMap,'+
            ' (select count(*) from messages m, folders f'+
            ' where m.folder_id=f.id and f.account_id=a.id'+
            ' and (flags @> array[\'pinned\',\'unread\']::varchar(255)[])) pinned_unread_count'+
            ' from accounts a'
        )).rows;
        const folders = (await this.pg.query(
            'select id, account_id, name,'+
            ' (select count(*) from messages m where m.folder_id=f.id) total_count,'+
            ' (select count(*) from messages m where m.folder_id=f.id and (flags @> array[\'unread\']::varchar(255)[])) unread_count'+
            ' from folders f order by account_id, name'
        )).rows;
        let fh = {};
        for (let i = 0; i < folders.length; i++)
        {
            fh[folders[i].account_id] = fh[folders[i].account_id] || [];
            fh[folders[i].account_id].push(folders[i]);
        }
        for (let i = 0; i < accounts.length; i++)
        {
            accounts[i].folders = fh[accounts[i].id] || [];
        }
        return res.send({ accounts: accounts });
    }

    msgSearchCond(query)
    {
        let p = {};
        if (query.folderId)
        {
            p['m.folder_id'] = query.folderId;
        }
        else if (query.folderType == 'unread')
        {
            p['(flags @> array[\'unread\']::varchar(255)[])'] = [];
        }
        else if (query.folderType == 'pinned')
        {
            p['(flags @> array[\'flagged\']::varchar(255)[])'] = [];
        }
        else if (query.folderType == 'inbox')
        {
            let folders = Object.keys(this.syncer.accounts)
                .map(id => [ id, this.syncer.accounts[id].settings.folders.spam ])
                .filter(f => f[1]);
            p['(f.account_id, f.name) NOT IN ('+folders.map(f => '(?, ?)').join(', ')+')'] =
                [].concat.apply([], folders);
            p['(flags @> array[\'in\']::varchar(255)[])'] = [];
        }
        else if (query.folderType == 'out')
        {
            p['(flags @> array[\'out\']::varchar(255)[])'] = [];
        }
        else if (query.folderType == 'outbox')
        {
            // TODO это какая-то хитрая метапапка, которая не живёт на IMAP'е?
        }
        else if (query.folderType == 'drafts' || query.folderType == 'spam' || query.folderType == 'trash')
        {
            let folders = Object.keys(this.syncer.accounts)
                .map(id => [ id, this.syncer.accounts[id].settings.folders[query.folderType] ])
                .filter(f => f[1]);
            p['(f.account_id, f.name) IN ('+folders.map(f => '(?, ?)').join(', ')+')'] =
                [].concat.apply([], folders);
        }
        if (typeof query.search == 'string' && query.search.trim())
        {
            p['messages_fulltext(m) @@ plainto_tsquery(?)'] = [ query.search.trim() ];
        }
        if (query.accountId)
        {
            p['f.account_id'] = query.accountId;
        }
        return Object.keys(p).length ? p : null;
    }

    async get_groups(req, res)
    {
        if (this.cfg.login && (!req.session || !req.session.auth))
        {
            return res.sendStatus(401);
        }
        const cond = this.msgSearchCond(req.query);
        if (!cond)
        {
            return res.status(500).send('Need message query parameters');
        }
        let intervals = [];
        let today, today_ts;
        today = new Date(ymd(new Date()));
        today_ts = today.getTime();
        let week_start = today_ts - ((today.getDay()+6)%7)*86400000;
        let prev_week = ymd(new Date(week_start - 86400000*7));
        for (let i = 1; i <= 12; i++)
        {
            let d = today.getFullYear()+'-'+(i < 10 ? '0' : '')+i+'-01';
            if (d >= prev_week)
                break;
            intervals.push({ date: d, name: 'm'+i });
        }
        intervals.push({ date: prev_week, name: 'pw' });
        for (let i = week_start, d = 1; i < today_ts; i += 86400000, d++)
        {
            intervals.push({ date: ymd(new Date(i)), name: 'd'+d });
        }
        for (let i = today.getFullYear()-1; i >= 1970; i--)
        {
            intervals.unshift({ date: i+'-01-01', name: ''+i });
        }
        intervals.push({ date: ymd(today), name: 't' });
        for (let i = 0; i < intervals.length-1; i++)
        {
            intervals[i].date_end = intervals[i+1].date;
        }
        intervals[intervals.length-1].date_end = '100000-12-31'; // it's faster than (is null or <)
        let groups = await SQL.select(
            this.pg,
            { d: SQL.values(intervals) },
            'd.name, d.date, ('+SQL.select_builder(
                { m: 'messages', f: [ 'INNER', 'folders', [ 'f.id=m.folder_id' ] ] },
                'count(*) count',
                { ...cond, 'm.time >= d.date::date and m.time < d.date_end::date': [] },
            )+') count',
            [],
            { order_by: 'date desc' }
        );
        groups = groups.filter(g => g.count > 0);
        return res.send({ groups: groups });
    }

    async get_messages(req, res)
    {
        if (this.cfg.login && (!req.session || !req.session.auth))
        {
            return res.sendStatus(401);
        }
        let cond = this.msgSearchCond(req.query);
        if (!cond)
        {
            return res.status(500).send('Need message query parameters');
        }
        let limit = req.query.limit || 50;
        if (limit > MAX_FETCH)
            limit = MAX_FETCH;
        let offset = req.query.offset || 0;
        let msgs = await SQL.select(
            this.pg,
            { m: 'messages', f: [ 'INNER', 'folders', [ 'f.id=m.folder_id' ] ] },
            'm.*', cond,
            { order_by: 'time desc', limit, offset }
        );
        for (let i = 0; i < msgs.length; i++)
        {
            delete msgs[i].text_index;
        }
        return res.send({ messages: msgs });
    }

    async get_message(req, res)
    {
        if (this.cfg.login && (!req.session || !req.session.auth))
        {
            return res.sendStatus(401);
        }
        let msgId = req.query.msgId;
        console.log('fetch message '+msgId);
        let msg = await SQL.select(
            this.pg,
            { m: 'messages', f: [ 'INNER', 'folders', [ 'f.id=m.folder_id' ] ] },
            'm.*, f.name folder_name, f.account_id',
            { 'm.id': msgId }, null, SQL.MS_ROW
        );
        delete msg.text_index;
        if (!msg)
        {
            return res.send({ error: 'not-found' });
        }
        if (!msg.body_html && !msg.body_text)
        {
            let srv = await this.syncer.imap.getConnection(msg.account_id, msg.folder_name);
            let upd = (await this.syncer.imap.runFetch(
                srv, msg.uid, { bodies: '' },
                async (messages, state) => await this.getBody(messages, msg.folder_id)
            ));
            this.syncer.imap.releaseConnection(msg.account_id);
            return res.send({ msg: { ...msg, ...upd[0] } });
        }
        return res.send({ msg: msg });
    }

    syncer_sync(params)
    {
        this.io.emit('sync', params);
    }

    async post_sync(req, res)
    {
        if (this.cfg.login && (!req.session || !req.session.auth))
        {
            return res.sendStatus(401);
        }
        if (this.syncer.syncInProgress)
        {
            return res.send({ error: 'already-running' });
        }
        this.syncer.syncAll().catch(console.error);
        return res.send({ status: 'started' });
    }

    async getBody(messages, boxId)
    {
        for (let i = 0; i < messages.length; i++)
        {
            let msg = messages[i];
            let obj = await this.syncer.parseMsg(msg[0].headers);
            obj.html = sanitizeHtml(obj.html);
            let upd = { body_text: obj.text||'', body_html: obj.html };
            upd.body_html_text = obj.html.replace(/<style[^>]*>.*<\/style\s*>|<\/?[^>]*>/g, '');
            await SQL.update(this.pg, 'messages m', upd, { folder_id: boxId, uid: msg[0].uid });
            if (messages.length == 1)
            {
                return [ upd ];
            }
        }
        return null;
    }
}

function rewriteCss(ast)
{
    var rules = ast.rules || ast.stylesheet && ast.stylesheet.rules;
    if (ast.stylesheet && ast.stylesheet.parsingErrors)
    {
        delete ast.stylesheet.parsingErrors;
    }
    if (rules)
    {
        for (var i = 0; i < rules.length; i++)
        {
            if (rules[i].type == 'document')
            {
                // prune @document instructions (may spy on current URL)
                rules.splice(i--, 1);
            }
            else if (rules[i].type == 'rule' && (!rules[i].selectors || !rules[i].declarations))
                rules.splice(i--, 1);
            else
                rewriteCss(rules[i]);
        }
    }
    else if (ast.type == 'rule')
    {
        for (var i = 0; i < ast.selectors.length; i++)
        {
            // FIXME: Do not hardcode css selector for frontend here
            // This will require generating unique substitution string,
            // so we may also generate 'blocked images' stubs when we do it.
            ast.selectors[i] = '.message-view .text '+ast.selectors[i];
        }
    }
}

function sanitizeHtml(html)
{
    let styles = '';
    html = (html||'').replace(/<style[^<>]*>([\s\S]*?)<\/style\s*>/ig, function(m, m1)
    {
        styles += m1+'\n';
        return '';
    });
    html = html.replace(/^[\s\S]*?<body[^<>]*>([\s\S]*)<\/body>[\s\S]*$/i, '$1');
    html = html.replace(/^[\s\S]*?<html[^<>]*>([\s\S]*)<\/html>[\s\S]*$/i, '$1');
    if (styles)
    {
        html = '<style>\n'+styles+'</style>\n'+html;
        styles = '';
    }
    html = htmlawed.sanitize(html||'', { safe: 1, elements: '* +style', keep_bad: 0, comment: 1 });
    html = html.replace(/<style[^>]*>([\s\S]*)<\/style\s*>/ig, function(m, m1)
    {
        let ast = css.parse(m1, { silent: true });
        rewriteCss(ast);
        return '<style>'+css.stringify(ast)+'</style>';
    });
    return html;
}

function wrapAsync(self, fn)
{
    return (req, res) => self[fn](req, res).catch(e => res.status(500).send('Internal Error: '+e.stack));
}

function ymd(dt)
{
    let m = dt.getMonth()+1;
    let d = dt.getDate();
    return dt.getFullYear()+'-'+(m < 10 ? '0'+m : m)+'-'+(d < 10 ? '0'+d : d);
}

module.exports = SyncerWeb;