diff --git a/ImapManager.js b/ImapManager.js index 7f1c5ad..dc86cfb 100644 --- a/ImapManager.js +++ b/ImapManager.js @@ -166,7 +166,7 @@ class ImapManager end(); } }) - .catch(e => reject(e)); + .catch(reject); }); f.once('end', () => diff --git a/Syncer.js b/Syncer.js index 4ff4a4f..1bdcce9 100644 --- a/Syncer.js +++ b/Syncer.js @@ -1,9 +1,14 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); + const Imap = require('imap'); const EventEmitter = require('events').EventEmitter; const iconv = require('iconv-lite'); const MailParser = require('mailparser').MailParser; const mimelib = require('mimelib'); +const fsp = require('./fsp.js'); const ImapManager = require('./ImapManager.js'); const sanitizeHtml = require('./sanitize.js'); const SQL = require('./select-builder-pgsql.js'); @@ -23,7 +28,16 @@ class Syncer // public async init(cfg) { - for (var i = 0; i < cfg.accounts.length; i++) + this.files_path = path.resolve(cfg.files_path); + try + { + fs.accessSync(this.files_path, fs.constants.R_OK || fs.constants.W_OK); + } + catch (e) + { + throw new Error(this.files_path+' is not writable'); + } + for (let i = 0; i < cfg.accounts.length; i++) { await this.addAccount(cfg.accounts[i]); } @@ -402,15 +416,60 @@ class Syncer } } - async parseMsg(msg) + async parseMsg(msg_text) { let parser = new MailParser({ streamAttachments: false, defaultCharset: 'windows-1251' }); - return await new Promise((r, j) => + let msg = await new Promise((resolve, reject) => { - parser.once('end', r); - parser.write(msg); + parser.on('error', reject); + parser.once('end', resolve); + parser.write(msg_text); parser.end(); }); + let byid = {}; + for (let a of msg.attachments||[]) + { + byid[a.contentId||''] = a; + } + msg.html = msg.html.replace(/(]*src=["']?)cid:([^'"\s]{1,256})/g, (m, m1, m2) => + { + if (!byid[m2]) + { + return m1 + 'cid:' + m2; + } + return m1 + 'data:' + byid[m2].contentType + ';base64,' + byid[m2].toString('base64'); + }); + let attachments = []; + for (let a of msg.attachments||[]) + { + let hash = crypto.createHash('sha1'); + hash.update(a.content); + let sha1 = hash.digest('hex'); + let subdir = sha1.substr(0, 2)+'/'+sha1.substr(2, 2); + let filename = subdir+'/'+sha1+'.bin'; + if (!await fsp.exists(this.files_path+'/'+filename)) + { + if (!await fsp.exists(this.files_path+'/'+sha1.substr(0, 2))) + { + await fsp.mkdir(this.files_path+'/'+sha1.substr(0, 2)); + } + if (!await fsp.exists(this.files_path+'/'+subdir)) + { + await fsp.mkdir(this.files_path+'/'+subdir); + } + await fsp.writeFile(this.files_path+'/'+filename, a.content); + } + attachments.push({ + id: a.contentId, + name: a.fileName, + mimetype: a.contentType, + size: a.length, + sha1, + filename, + }); + } + msg.attachments = attachments; + return msg; } extractAttachments(struct, attachments) @@ -424,11 +483,11 @@ class Syncer } else if (struct[i].disposition && struct[i].disposition.type == 'attachment') { - attachments.push([ - mimelib.parseMimeWords(struct[i].disposition.params && struct[i].disposition.params.filename || struct[i].description || ''), - struct[i].type+'/'+struct[i].subtype, - struct[i].size, - ]); + attachments.push({ + name: mimelib.parseMimeWords(struct[i].disposition.params && struct[i].disposition.params.filename || struct[i].description || ''), + mimetype: struct[i].type+'/'+struct[i].subtype, + size: struct[i].size, + }); } } return attachments; @@ -538,12 +597,10 @@ class Syncer async fetchFullMessage(account_id, folder_id, folder_name, msg_uid) { - // FIXME: parse and save attachments - // FIXME: replace inline images let srv = await this.imap.getConnection(account_id, folder_name); let upd = await this.imap.runFetch( srv, msg_uid, { bodies: '' }, - async(messages, state) => await this._parseBody(messages, folder_id) + (messages, state) => this._parseBody(messages, folder_id) ); this.imap.releaseConnection(account_id); return upd; @@ -556,11 +613,20 @@ class Syncer let msg = messages[i]; let obj = await this.parseMsg(msg[0].headers); obj.html = sanitizeHtml(obj.html); - let upd = { body_text: obj.text||'', body_html: obj.html }; - upd.body_html_text = obj.html.replace(/]*>.*<\/style\s*>|<\/?[^>]*>/g, ''); - await SQL.update(this.pg, 'messages m', upd, { folder_id: boxId, uid: msg[0].uid }); + let upd = { + body_text: obj.text||'', + body_html: obj.html, + body_html_text: obj.html.replace(/]*>.*<\/style\s*>|<\/?[^>]*>/g, ''), + }; + /*await SQL.update( + this.pg, 'messages m', { + ...upd, + 'props = props || ?': [ { attachments: obj.attachments } ] + }, { folder_id: boxId, uid: msg[0].uid } + );*/ if (messages.length == 1) { + upd.props = { attachments: obj.attachments }; return [ upd ]; } } diff --git a/SyncerWeb.js b/SyncerWeb.js index bc7a59e..82e51bc 100644 --- a/SyncerWeb.js +++ b/SyncerWeb.js @@ -251,7 +251,7 @@ class SyncerWeb if (!msg.body_html && !msg.body_text) { let upd = await this.syncer.fetchFullMessage(msg.account_id, msg.folder_id, msg.folder_name, msg.uid); - return res.send({ msg: { ...msg, ...upd[0] } }); + return res.send({ msg: { ...msg, ...upd[0], props: { ...msg.props, ...upd[0].props } } }); } return res.send({ msg: msg }); } diff --git a/database-patches/2019-05-18-attachments.sql b/database-patches/2019-05-18-attachments.sql new file mode 100644 index 0000000..ca9a3cd --- /dev/null +++ b/database-patches/2019-05-18-attachments.sql @@ -0,0 +1,27 @@ +begin; + +create or replace function messages_fulltext(msg messages) returns tsvector +language plpgsql immutable as $$ +begin + return setweight(to_tsvector('russian', regexp_replace( + coalesce(msg.props->>'from', '') || ' ' || + coalesce(msg.props->>'replyto', '') || ' ' || + coalesce(msg.props->>'to', '') || ' ' || + coalesce(msg.props->>'cc', '') || ' ' || + coalesce(msg.props->>'bcc', '') || ' ' || + (select string_agg((a->>'name') || ' ' || (a->>'mimetype') || ' ' || (a->>'size'), ' ') + from jsonb_array_elements(coalesce(msg.props->'attachments', '[]'::jsonb)) as t (a)) || ' ' || + msg.subject, + '\W+', ' ', 'g' + )), 'A') + || setweight(to_tsvector('russian', msg.body_html_text || ' ' || msg.body_text), 'B'); +end +$$; + +update messages +set props = props || jsonb_build_object('attachments', ( + select jsonb_agg(jsonb_build_object('name', a->>0, 'mimetype', a->>1, 'size', a->>2)) from jsonb_array_elements(props->'attachments') as t (a) +)) +where jsonb_array_length(props->'attachments') > 0; + +commit; diff --git a/database/db.sql b/database/db.sql index 1d5d50d..69bc6f0 100644 --- a/database/db.sql +++ b/database/db.sql @@ -82,7 +82,8 @@ begin coalesce(msg.props->>'to', '') || ' ' || coalesce(msg.props->>'cc', '') || ' ' || coalesce(msg.props->>'bcc', '') || ' ' || - coalesce(msg.props->>'attachments', '') || ' ' || + (select string_agg((a->>'name') || ' ' || (a->>'mimetype') || ' ' || (a->>'size'), ' ') + from jsonb_array_values(coalesce(msg.props->'attachments', '[]'::jsonb)) as t (a)) || ' ' || msg.subject, '\W+', ' ', 'g' )), 'A') diff --git a/fsp.js b/fsp.js new file mode 100644 index 0000000..cb136e0 --- /dev/null +++ b/fsp.js @@ -0,0 +1,43 @@ +const fs = require('fs'); + +module.exports = { + async writeFile(path, content, options) + { + return await new Promise((ok, no) => + { + fs.writeFile(path, content, options, err => err ? no(err) : ok()); + }); + }, + + async rename(from, to) + { + return await new Promise((ok, no) => + { + fs.rename(from, to, err => err ? no(err) : ok()); + }); + }, + + async mkdir(path, options) + { + return await new Promise((ok, no) => + { + fs.mkdir(path, options, err => err ? no(err) : ok()); + }); + }, + + async exists(path) + { + return await new Promise((ok, no) => + { + fs.access(path, fs.constants.R_OK, err => err ? (err.code == 'ENOENT' ? ok(false) : no(err)) : ok(true)); + }); + }, + + async is_writable(path) + { + return await new Promise((ok, no) => + { + fs.access(path, fs.constants.R_OK | fs.constants.W_OK, err => ok(!err)); + }); + }, +}; diff --git a/operetta.js b/operetta.js index f96ec0c..9f9c420 100644 --- a/operetta.js +++ b/operetta.js @@ -61,7 +61,7 @@ async function startSync(cfg) let syncerweb = new SyncerWeb(syncer, dbh, cfg); await syncer.init(cfg); syncerweb.listen(8057); - await syncer.syncAll(); + //await syncer.syncAll(); } let cfg = require('./cfg.json');