diff --git a/python/pyphantomjs/arguments.py b/python/pyphantomjs/arguments.py index 46034d7f..db7ec862 100644 --- a/python/pyphantomjs/arguments.py +++ b/python/pyphantomjs/arguments.py @@ -18,11 +18,17 @@ ''' import argparse +import codecs +import os +import sys +from PyQt4.QtCore import qInstallMsgHandler, QObject, qWarning from PyQt4.QtNetwork import QNetworkProxy +from PyQt4.QtWebKit import QWebPage from __init__ import __version__ from plugincontroller import do_action +from utils import debug, MessageHandler, QPyFile license = ''' @@ -69,8 +75,15 @@ def argParser(): answer = True if value == 'yes' else False setattr(namespace, self.dest, answer) + def proxyType(type_): + if type_ == QNetworkProxy.HttpProxy: + return 'http' + elif type_ == QNetworkProxy.Socks5Proxy: + return 'socks5' + yesOrNo = lambda d: 'yes' if d else 'no' + parser = argparse.ArgumentParser( description='Minimalistic headless WebKit-based JavaScript-driven tool', usage='%(prog)s [options] script.[js|coffee] [script argument [script argument ...]]', @@ -110,7 +123,7 @@ def argParser(): help='Set the network proxy' ) program.add_argument('--proxy-type', default=defaults['proxyType'], metavar='type', - help='Set the network proxy type (default: http)' + help='Set the network proxy type (default: %s)' % proxyType(defaults['proxyType']) ) program.add_argument('--script-encoding', default=defaults['scriptEncoding'], metavar='encoding', help='Sets the encoding used for scripts (default: %(default)s)' @@ -144,3 +157,104 @@ def argParser(): do_action('ArgParser') return parser + + +def parseArgs(app, args): + # Handle all command-line options + p = argParser() + arg_data = p.parse_known_args(args) + args = arg_data[0] + args.script_args = arg_data[1] + + # register an alternative Message Handler + messageHandler = MessageHandler(args.verbose) + qInstallMsgHandler(messageHandler.process) + + file_check = (args.cookies_file, args.config) + for file_ in file_check: + if file_ is not None and not os.path.exists(file_): + sys.exit("No such file or directory: '%s'" % file_) + + if args.config: + config = Config(app, args.config) + # apply settings + for setting in config.settings: + setattr(args, config.settings[setting]['mapping'], config.property(setting)) + + split_check = ( + (args.proxy, 'proxy'), + ) + for arg, name in split_check: + if arg: + item = arg.split(':') + if len(item) < 2 or not len(item[1]): + p.print_help() + sys.exit(1) + setattr(args, name, item) + + if args.proxy is not None: + if args.proxy_type == 'socks5': + args.proxy_type = QNetworkProxy.Socks5Proxy + + do_action('ParseArgs', args) + + if args.debug: + debug(args.debug) + + # verbose flag got changed on us, so we reload the flag + if messageHandler.verbose != args.verbose: + messageHandler.verbose = args.verbose + + if args.script is None: + p.print_help() + sys.exit(1) + + if not os.path.exists(args.script): + sys.exit("No such file or directory: '%s'" % args.script) + + return args + + +class Config(QObject): + def __init__(self, parent, jsonFile): + super(Config, self).__init__(parent) + + with codecs.open(jsonFile, encoding='utf-8') as f: + json = f.read() + + self.settings = { + 'cookiesFile': { 'mapping': 'cookies_file', 'default': defaults['cookiesFile'] }, + 'debug': { 'mapping': 'debug', 'default': defaults['debug'] }, + 'diskCache': { 'mapping': 'disk_cache', 'default': defaults['diskCache'] }, + 'ignoreSslErrors': { 'mapping': 'ignore_ssl_errors', 'default': defaults['ignoreSslErrors'] }, + 'loadImages': { 'mapping': 'load_images', 'default': defaults['loadImages'] }, + 'loadPlugins': { 'mapping': 'load_plugins', 'default': defaults['loadPlugins'] }, + 'localToRemoteUrlAccessEnabled': { 'mapping': 'local_to_remote_url_access', 'default': defaults['localToRemoteUrlAccessEnabled'] }, + 'maxDiskCacheSize': { 'mapping': 'max_disk_cache_size', 'default': defaults['maxDiskCacheSize'] }, + 'outputEncoding': { 'mapping': 'output_encoding', 'default': defaults['outputEncoding'] }, + 'proxy': { 'mapping': 'proxy', 'default': defaults['proxy'] }, + 'proxyType': { 'mapping': 'proxy_type', 'default': defaults['proxyType'] }, + 'scriptEncoding': { 'mapping': 'script_encoding', 'default': defaults['scriptEncoding'] }, + 'verbose': { 'mapping': 'verbose', 'default': defaults['verbose'] } + } + + do_action('ConfigInit', self.settings) + + # generate dynamic properties + for setting in self.settings: + self.setProperty(setting, self.settings[setting]['default']) + + # now it's time to parse our JSON file + if not json.lstrip().startswith('{') or not json.rstrip().endswith('}'): + qWarning('Config file MUST be in JSON format!') + return + + webPage = QWebPage(self) + + with QPyFile(':/configurator.js') as f: + # add config object + webPage.mainFrame().addToJavaScriptWindowObject('config', self) + # apply settings + webPage.mainFrame().evaluateJavaScript(f.readAll().replace('%1', json)) + + do_action('Config') diff --git a/python/pyphantomjs/config.py b/python/pyphantomjs/config.py deleted file mode 100644 index 0a6bd6dc..00000000 --- a/python/pyphantomjs/config.py +++ /dev/null @@ -1,73 +0,0 @@ -''' - This file is part of the PyPhantomJS project. - - Copyright (C) 2011 James Roe - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' - -import codecs -import sys - -from PyQt4.QtCore import QObject, qWarning -from PyQt4.QtWebKit import QWebPage - -from arguments import defaults -from plugincontroller import do_action -from utils import QPyFile - - -class Config(QObject): - def __init__(self, parent, jsonFile): - super(Config, self).__init__(parent) - - with codecs.open(jsonFile, encoding='utf-8') as f: - json = f.read() - - self.settings = { - 'cookiesFile': { 'mapping': 'cookies_file', 'default': defaults['cookiesFile'] }, - 'debug': { 'mapping': 'debug', 'default': defaults['debug'] }, - 'diskCache': { 'mapping': 'disk_cache', 'default': defaults['diskCache'] }, - 'ignoreSslErrors': { 'mapping': 'ignore_ssl_errors', 'default': defaults['ignoreSslErrors'] }, - 'loadImages': { 'mapping': 'load_images', 'default': defaults['loadImages'] }, - 'loadPlugins': { 'mapping': 'load_plugins', 'default': defaults['loadPlugins'] }, - 'localToRemoteUrlAccessEnabled': { 'mapping': 'local_to_remote_url_access', 'default': defaults['localToRemoteUrlAccessEnabled'] }, - 'maxDiskCacheSize': { 'mapping': 'max_disk_cache_size', 'default': defaults['maxDiskCacheSize'] }, - 'outputEncoding': { 'mapping': 'output_encoding', 'default': defaults['outputEncoding'] }, - 'proxy': { 'mapping': 'proxy', 'default': defaults['proxy'] }, - 'proxyType': { 'mapping': 'proxy_type', 'default': defaults['proxyType'] }, - 'scriptEncoding': { 'mapping': 'script_encoding', 'default': defaults['scriptEncoding'] }, - 'verbose': { 'mapping': 'verbose', 'default': defaults['verbose'] } - } - - do_action('ConfigInit', self.settings) - - # generate dynamic properties - for setting in self.settings: - self.setProperty(setting, self.settings[setting]['default']) - - # now it's time to parse our JSON file - if not json.lstrip().startswith('{') or not json.rstrip().endswith('}'): - qWarning('Config file MUST be in JSON format!') - return - - webPage = QWebPage(self) - - with QPyFile(':/configurator.js') as f: - # add config object - webPage.mainFrame().addToJavaScriptWindowObject('config', self) - # apply settings - webPage.mainFrame().evaluateJavaScript(f.readAll().replace('%1', json)) - - do_action('Config') diff --git a/python/pyphantomjs/phantom.py b/python/pyphantomjs/phantom.py index 8fe56a4d..46f98209 100644 --- a/python/pyphantomjs/phantom.py +++ b/python/pyphantomjs/phantom.py @@ -29,8 +29,8 @@ from __init__ import __version_info__ from encoding import Encode from filesystem import FileSystem from plugincontroller import do_action -from utils import injectJsInFrame, QPyFile -from webpage import WebPage +from utils import QPyFile +from webpage import injectJsInFrame, WebPage class Phantom(QObject): diff --git a/python/pyphantomjs/pyphantomjs.py b/python/pyphantomjs/pyphantomjs.py index c287ae37..d17c52ef 100644 --- a/python/pyphantomjs/pyphantomjs.py +++ b/python/pyphantomjs/pyphantomjs.py @@ -24,12 +24,9 @@ for item in ('QDate', 'QDateTime', 'QString', 'QTextStream', 'QTime' 'QUrl', 'QVariant'): sip.setapi(item, 2) -import os import sys -from PyQt4.QtCore import qInstallMsgHandler from PyQt4.QtGui import QApplication, QIcon -from PyQt4.QtNetwork import QNetworkProxy from plugincontroller import do_action # load plugins if running script directly @@ -39,10 +36,8 @@ if __name__ == '__main__': import resources from __init__ import __version__ -from arguments import argParser -from config import Config +from arguments import parseArgs from phantom import Phantom -from utils import MessageHandler # make keyboard interrupt quit program import signal @@ -54,85 +49,6 @@ sys.stdout = SafeStreamFilter(sys.stdout) sys.stderr = SafeStreamFilter(sys.stderr) -def debug(debug_type): - def excepthook(type_, value, tb): - import traceback - - # print the exception... - traceback.print_exception(type_, value, tb) - print - # ...then start the debugger in post-mortem mode - pdb.pm() - - # we are NOT in interactive mode - if not hasattr(sys, 'ps1') or sys.stderr.target.isatty(): - import pdb - - from PyQt4.QtCore import pyqtRemoveInputHook - pyqtRemoveInputHook() - - if debug_type == 'exception': - sys.excepthook = excepthook - elif debug_type == 'program': - pdb.set_trace() - - -def parseArgs(app, args): - # Handle all command-line options - p = argParser() - arg_data = p.parse_known_args(args) - args = arg_data[0] - args.script_args = arg_data[1] - - # register an alternative Message Handler - messageHandler = MessageHandler(args.verbose) - qInstallMsgHandler(messageHandler.process) - - file_check = (args.cookies_file, args.config) - for file_ in file_check: - if file_ is not None and not os.path.exists(file_): - sys.exit("No such file or directory: '%s'" % file_) - - if args.config: - config = Config(app, args.config) - # apply settings - for setting in config.settings: - setattr(args, config.settings[setting]['mapping'], config.property(setting)) - - split_check = ( - (args.proxy, 'proxy'), - ) - for arg, name in split_check: - if arg: - item = arg.split(':') - if len(item) < 2 or not len(item[1]): - p.print_help() - sys.exit(1) - setattr(args, name, item) - - if args.proxy is not None: - if args.proxy_type == 'socks5': - args.proxy_type = QNetworkProxy.Socks5Proxy - - do_action('ParseArgs', args) - - if args.debug: - debug(args.debug) - - # verbose flag got changed on us, so we reload the flag - if messageHandler.verbose != args.verbose: - messageHandler.verbose = args.verbose - - if args.script is None: - p.print_help() - sys.exit(1) - - if not os.path.exists(args.script): - sys.exit("No such file or directory: '%s'" % args.script) - - return args - - def main(): app = QApplication(sys.argv) diff --git a/python/pyphantomjs/utils.py b/python/pyphantomjs/utils.py index 078bedab..fb7c3479 100644 --- a/python/pyphantomjs/utils.py +++ b/python/pyphantomjs/utils.py @@ -17,54 +17,34 @@ along with this program. If not, see . ''' -import codecs -import os import sys -from PyQt4.QtCore import (QByteArray, QDateTime, qDebug, QFile, Qt, +from PyQt4.QtCore import (QByteArray, QDateTime, QFile, Qt, QtCriticalMsg, QtDebugMsg, QtFatalMsg, QtWarningMsg) -CSConverter = None -def coffee2js(script): - global CSConverter - if not CSConverter: - from csconverter import CSConverter - return CSConverter().convert(script) +def debug(debug_type): + def excepthook(type_, value, tb): + import traceback + # print the exception... + traceback.print_exception(type_, value, tb) + print + # ...then start the debugger in post-mortem mode + pdb.pm() -def injectJsInFrame(filePath, scriptEncoding, libraryPath, targetFrame, startingScript=False): - try: - # if file doesn't exist in the CWD, use the lookup - if not os.path.exists(filePath): - filePath = os.path.join(libraryPath, filePath) + # we are NOT in interactive mode + if not hasattr(sys, 'ps1') or sys.stderr.target.isatty(): + import pdb - try: - with codecs.open(filePath, encoding=scriptEncoding) as f: - script = f.read() - except UnicodeDecodeError as e: - sys.exit("%s in '%s'" % (e, filePath)) + from PyQt4.QtCore import pyqtRemoveInputHook + pyqtRemoveInputHook() - if script.startswith('#!') and not filePath.lower().endswith('.coffee'): - script = '//' + script - - if filePath.lower().endswith('.coffee'): - result = coffee2js(script) - if not result[0]: - if startingScript: - sys.exit("%s: '%s'" % (result[1], filePath)) - else: - qDebug("%s: '%s'" % (result[1], filePath)) - script = '' - else: - script = result[1] - - targetFrame.evaluateJavaScript(script) - return True - except IOError as (t, e): - qDebug("%s: '%s'" % (e, filePath)) - return False + if debug_type == 'exception': + sys.excepthook = excepthook + elif debug_type == 'program': + pdb.set_trace() class MessageHandler(object): diff --git a/python/pyphantomjs/webpage.py b/python/pyphantomjs/webpage.py index b768a006..7bdc7080 100644 --- a/python/pyphantomjs/webpage.py +++ b/python/pyphantomjs/webpage.py @@ -17,6 +17,9 @@ along with this program. If not, see . ''' +import codecs +import os +import sys from cStringIO import StringIO from math import ceil, floor @@ -36,9 +39,42 @@ try: except ImportError: qDebug('PIL not found! Saving to gif files will be disabled.') +from csconverter import CSConverter from networkaccessmanager import NetworkAccessManager from plugincontroller import do_action -from utils import injectJsInFrame + + +def injectJsInFrame(filePath, scriptEncoding, libraryPath, targetFrame, startingScript=False): + try: + # if file doesn't exist in the CWD, use the lookup + if not os.path.exists(filePath): + filePath = os.path.join(libraryPath, filePath) + + try: + with codecs.open(filePath, encoding=scriptEncoding) as f: + script = f.read() + except UnicodeDecodeError as e: + sys.exit("%s in '%s'" % (e, filePath)) + + if script.startswith('#!') and not filePath.lower().endswith('.coffee'): + script = '//' + script + + if filePath.lower().endswith('.coffee'): + result = CSConverter().convert(script) + if not result[0]: + if startingScript: + sys.exit("%s: '%s'" % (result[1], filePath)) + else: + qDebug("%s: '%s'" % (result[1], filePath)) + script = '' + else: + script = result[1] + + targetFrame.evaluateJavaScript(script) + return True + except IOError as (t, e): + qDebug("%s: '%s'" % (e, filePath)) + return False class CustomPage(QWebPage):