mirror of https://github.com/vitalif/phantomjs
Further refactoring
Modified utils a bit to remove the problem of future circular imports Cleaned up main pyphantomjs file Cleaned up a little of the code1.4
IceArmy
parent
4ec8df3a84
commit
424214361a
|
|
@ -18,11 +18,17 @@
|
|||
'''
|
||||
|
||||
import argparse
|
||||
import codecs
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PyQt4.QtCore import qInstallMsgHandler, QObject, qWarning
|
||||
from PyQt4.QtNetwork import QNetworkProxy
|
||||
from PyQt4.QtWebKit import QWebPage
|
||||
|
||||
from __init__ import __version__
|
||||
from plugincontroller import do_action
|
||||
from utils import debug, MessageHandler, QPyFile
|
||||
|
||||
|
||||
license = '''
|
||||
|
|
@ -69,8 +75,15 @@ def argParser():
|
|||
answer = True if value == 'yes' else False
|
||||
setattr(namespace, self.dest, answer)
|
||||
|
||||
def proxyType(type_):
|
||||
if type_ == QNetworkProxy.HttpProxy:
|
||||
return 'http'
|
||||
elif type_ == QNetworkProxy.Socks5Proxy:
|
||||
return 'socks5'
|
||||
|
||||
yesOrNo = lambda d: 'yes' if d else 'no'
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Minimalistic headless WebKit-based JavaScript-driven tool',
|
||||
usage='%(prog)s [options] script.[js|coffee] [script argument [script argument ...]]',
|
||||
|
|
@ -110,7 +123,7 @@ def argParser():
|
|||
help='Set the network proxy'
|
||||
)
|
||||
program.add_argument('--proxy-type', default=defaults['proxyType'], metavar='type',
|
||||
help='Set the network proxy type (default: http)'
|
||||
help='Set the network proxy type (default: %s)' % proxyType(defaults['proxyType'])
|
||||
)
|
||||
program.add_argument('--script-encoding', default=defaults['scriptEncoding'], metavar='encoding',
|
||||
help='Sets the encoding used for scripts (default: %(default)s)'
|
||||
|
|
@ -144,3 +157,104 @@ def argParser():
|
|||
do_action('ArgParser')
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def parseArgs(app, args):
|
||||
# Handle all command-line options
|
||||
p = argParser()
|
||||
arg_data = p.parse_known_args(args)
|
||||
args = arg_data[0]
|
||||
args.script_args = arg_data[1]
|
||||
|
||||
# register an alternative Message Handler
|
||||
messageHandler = MessageHandler(args.verbose)
|
||||
qInstallMsgHandler(messageHandler.process)
|
||||
|
||||
file_check = (args.cookies_file, args.config)
|
||||
for file_ in file_check:
|
||||
if file_ is not None and not os.path.exists(file_):
|
||||
sys.exit("No such file or directory: '%s'" % file_)
|
||||
|
||||
if args.config:
|
||||
config = Config(app, args.config)
|
||||
# apply settings
|
||||
for setting in config.settings:
|
||||
setattr(args, config.settings[setting]['mapping'], config.property(setting))
|
||||
|
||||
split_check = (
|
||||
(args.proxy, 'proxy'),
|
||||
)
|
||||
for arg, name in split_check:
|
||||
if arg:
|
||||
item = arg.split(':')
|
||||
if len(item) < 2 or not len(item[1]):
|
||||
p.print_help()
|
||||
sys.exit(1)
|
||||
setattr(args, name, item)
|
||||
|
||||
if args.proxy is not None:
|
||||
if args.proxy_type == 'socks5':
|
||||
args.proxy_type = QNetworkProxy.Socks5Proxy
|
||||
|
||||
do_action('ParseArgs', args)
|
||||
|
||||
if args.debug:
|
||||
debug(args.debug)
|
||||
|
||||
# verbose flag got changed on us, so we reload the flag
|
||||
if messageHandler.verbose != args.verbose:
|
||||
messageHandler.verbose = args.verbose
|
||||
|
||||
if args.script is None:
|
||||
p.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
if not os.path.exists(args.script):
|
||||
sys.exit("No such file or directory: '%s'" % args.script)
|
||||
|
||||
return args
|
||||
|
||||
|
||||
class Config(QObject):
|
||||
def __init__(self, parent, jsonFile):
|
||||
super(Config, self).__init__(parent)
|
||||
|
||||
with codecs.open(jsonFile, encoding='utf-8') as f:
|
||||
json = f.read()
|
||||
|
||||
self.settings = {
|
||||
'cookiesFile': { 'mapping': 'cookies_file', 'default': defaults['cookiesFile'] },
|
||||
'debug': { 'mapping': 'debug', 'default': defaults['debug'] },
|
||||
'diskCache': { 'mapping': 'disk_cache', 'default': defaults['diskCache'] },
|
||||
'ignoreSslErrors': { 'mapping': 'ignore_ssl_errors', 'default': defaults['ignoreSslErrors'] },
|
||||
'loadImages': { 'mapping': 'load_images', 'default': defaults['loadImages'] },
|
||||
'loadPlugins': { 'mapping': 'load_plugins', 'default': defaults['loadPlugins'] },
|
||||
'localToRemoteUrlAccessEnabled': { 'mapping': 'local_to_remote_url_access', 'default': defaults['localToRemoteUrlAccessEnabled'] },
|
||||
'maxDiskCacheSize': { 'mapping': 'max_disk_cache_size', 'default': defaults['maxDiskCacheSize'] },
|
||||
'outputEncoding': { 'mapping': 'output_encoding', 'default': defaults['outputEncoding'] },
|
||||
'proxy': { 'mapping': 'proxy', 'default': defaults['proxy'] },
|
||||
'proxyType': { 'mapping': 'proxy_type', 'default': defaults['proxyType'] },
|
||||
'scriptEncoding': { 'mapping': 'script_encoding', 'default': defaults['scriptEncoding'] },
|
||||
'verbose': { 'mapping': 'verbose', 'default': defaults['verbose'] }
|
||||
}
|
||||
|
||||
do_action('ConfigInit', self.settings)
|
||||
|
||||
# generate dynamic properties
|
||||
for setting in self.settings:
|
||||
self.setProperty(setting, self.settings[setting]['default'])
|
||||
|
||||
# now it's time to parse our JSON file
|
||||
if not json.lstrip().startswith('{') or not json.rstrip().endswith('}'):
|
||||
qWarning('Config file MUST be in JSON format!')
|
||||
return
|
||||
|
||||
webPage = QWebPage(self)
|
||||
|
||||
with QPyFile(':/configurator.js') as f:
|
||||
# add config object
|
||||
webPage.mainFrame().addToJavaScriptWindowObject('config', self)
|
||||
# apply settings
|
||||
webPage.mainFrame().evaluateJavaScript(f.readAll().replace('%1', json))
|
||||
|
||||
do_action('Config')
|
||||
|
|
|
|||
|
|
@ -1,73 +0,0 @@
|
|||
'''
|
||||
This file is part of the PyPhantomJS project.
|
||||
|
||||
Copyright (C) 2011 James Roe <roejames12@hotmail.com>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
'''
|
||||
|
||||
import codecs
|
||||
import sys
|
||||
|
||||
from PyQt4.QtCore import QObject, qWarning
|
||||
from PyQt4.QtWebKit import QWebPage
|
||||
|
||||
from arguments import defaults
|
||||
from plugincontroller import do_action
|
||||
from utils import QPyFile
|
||||
|
||||
|
||||
class Config(QObject):
|
||||
def __init__(self, parent, jsonFile):
|
||||
super(Config, self).__init__(parent)
|
||||
|
||||
with codecs.open(jsonFile, encoding='utf-8') as f:
|
||||
json = f.read()
|
||||
|
||||
self.settings = {
|
||||
'cookiesFile': { 'mapping': 'cookies_file', 'default': defaults['cookiesFile'] },
|
||||
'debug': { 'mapping': 'debug', 'default': defaults['debug'] },
|
||||
'diskCache': { 'mapping': 'disk_cache', 'default': defaults['diskCache'] },
|
||||
'ignoreSslErrors': { 'mapping': 'ignore_ssl_errors', 'default': defaults['ignoreSslErrors'] },
|
||||
'loadImages': { 'mapping': 'load_images', 'default': defaults['loadImages'] },
|
||||
'loadPlugins': { 'mapping': 'load_plugins', 'default': defaults['loadPlugins'] },
|
||||
'localToRemoteUrlAccessEnabled': { 'mapping': 'local_to_remote_url_access', 'default': defaults['localToRemoteUrlAccessEnabled'] },
|
||||
'maxDiskCacheSize': { 'mapping': 'max_disk_cache_size', 'default': defaults['maxDiskCacheSize'] },
|
||||
'outputEncoding': { 'mapping': 'output_encoding', 'default': defaults['outputEncoding'] },
|
||||
'proxy': { 'mapping': 'proxy', 'default': defaults['proxy'] },
|
||||
'proxyType': { 'mapping': 'proxy_type', 'default': defaults['proxyType'] },
|
||||
'scriptEncoding': { 'mapping': 'script_encoding', 'default': defaults['scriptEncoding'] },
|
||||
'verbose': { 'mapping': 'verbose', 'default': defaults['verbose'] }
|
||||
}
|
||||
|
||||
do_action('ConfigInit', self.settings)
|
||||
|
||||
# generate dynamic properties
|
||||
for setting in self.settings:
|
||||
self.setProperty(setting, self.settings[setting]['default'])
|
||||
|
||||
# now it's time to parse our JSON file
|
||||
if not json.lstrip().startswith('{') or not json.rstrip().endswith('}'):
|
||||
qWarning('Config file MUST be in JSON format!')
|
||||
return
|
||||
|
||||
webPage = QWebPage(self)
|
||||
|
||||
with QPyFile(':/configurator.js') as f:
|
||||
# add config object
|
||||
webPage.mainFrame().addToJavaScriptWindowObject('config', self)
|
||||
# apply settings
|
||||
webPage.mainFrame().evaluateJavaScript(f.readAll().replace('%1', json))
|
||||
|
||||
do_action('Config')
|
||||
|
|
@ -29,8 +29,8 @@ from __init__ import __version_info__
|
|||
from encoding import Encode
|
||||
from filesystem import FileSystem
|
||||
from plugincontroller import do_action
|
||||
from utils import injectJsInFrame, QPyFile
|
||||
from webpage import WebPage
|
||||
from utils import QPyFile
|
||||
from webpage import injectJsInFrame, WebPage
|
||||
|
||||
|
||||
class Phantom(QObject):
|
||||
|
|
|
|||
|
|
@ -24,12 +24,9 @@ for item in ('QDate', 'QDateTime', 'QString', 'QTextStream', 'QTime'
|
|||
'QUrl', 'QVariant'):
|
||||
sip.setapi(item, 2)
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PyQt4.QtCore import qInstallMsgHandler
|
||||
from PyQt4.QtGui import QApplication, QIcon
|
||||
from PyQt4.QtNetwork import QNetworkProxy
|
||||
|
||||
from plugincontroller import do_action
|
||||
# load plugins if running script directly
|
||||
|
|
@ -39,10 +36,8 @@ if __name__ == '__main__':
|
|||
|
||||
import resources
|
||||
from __init__ import __version__
|
||||
from arguments import argParser
|
||||
from config import Config
|
||||
from arguments import parseArgs
|
||||
from phantom import Phantom
|
||||
from utils import MessageHandler
|
||||
|
||||
# make keyboard interrupt quit program
|
||||
import signal
|
||||
|
|
@ -54,85 +49,6 @@ sys.stdout = SafeStreamFilter(sys.stdout)
|
|||
sys.stderr = SafeStreamFilter(sys.stderr)
|
||||
|
||||
|
||||
def debug(debug_type):
|
||||
def excepthook(type_, value, tb):
|
||||
import traceback
|
||||
|
||||
# print the exception...
|
||||
traceback.print_exception(type_, value, tb)
|
||||
print
|
||||
# ...then start the debugger in post-mortem mode
|
||||
pdb.pm()
|
||||
|
||||
# we are NOT in interactive mode
|
||||
if not hasattr(sys, 'ps1') or sys.stderr.target.isatty():
|
||||
import pdb
|
||||
|
||||
from PyQt4.QtCore import pyqtRemoveInputHook
|
||||
pyqtRemoveInputHook()
|
||||
|
||||
if debug_type == 'exception':
|
||||
sys.excepthook = excepthook
|
||||
elif debug_type == 'program':
|
||||
pdb.set_trace()
|
||||
|
||||
|
||||
def parseArgs(app, args):
|
||||
# Handle all command-line options
|
||||
p = argParser()
|
||||
arg_data = p.parse_known_args(args)
|
||||
args = arg_data[0]
|
||||
args.script_args = arg_data[1]
|
||||
|
||||
# register an alternative Message Handler
|
||||
messageHandler = MessageHandler(args.verbose)
|
||||
qInstallMsgHandler(messageHandler.process)
|
||||
|
||||
file_check = (args.cookies_file, args.config)
|
||||
for file_ in file_check:
|
||||
if file_ is not None and not os.path.exists(file_):
|
||||
sys.exit("No such file or directory: '%s'" % file_)
|
||||
|
||||
if args.config:
|
||||
config = Config(app, args.config)
|
||||
# apply settings
|
||||
for setting in config.settings:
|
||||
setattr(args, config.settings[setting]['mapping'], config.property(setting))
|
||||
|
||||
split_check = (
|
||||
(args.proxy, 'proxy'),
|
||||
)
|
||||
for arg, name in split_check:
|
||||
if arg:
|
||||
item = arg.split(':')
|
||||
if len(item) < 2 or not len(item[1]):
|
||||
p.print_help()
|
||||
sys.exit(1)
|
||||
setattr(args, name, item)
|
||||
|
||||
if args.proxy is not None:
|
||||
if args.proxy_type == 'socks5':
|
||||
args.proxy_type = QNetworkProxy.Socks5Proxy
|
||||
|
||||
do_action('ParseArgs', args)
|
||||
|
||||
if args.debug:
|
||||
debug(args.debug)
|
||||
|
||||
# verbose flag got changed on us, so we reload the flag
|
||||
if messageHandler.verbose != args.verbose:
|
||||
messageHandler.verbose = args.verbose
|
||||
|
||||
if args.script is None:
|
||||
p.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
if not os.path.exists(args.script):
|
||||
sys.exit("No such file or directory: '%s'" % args.script)
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
app = QApplication(sys.argv)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,54 +17,34 @@
|
|||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
'''
|
||||
|
||||
import codecs
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PyQt4.QtCore import (QByteArray, QDateTime, qDebug, QFile, Qt,
|
||||
from PyQt4.QtCore import (QByteArray, QDateTime, QFile, Qt,
|
||||
QtCriticalMsg, QtDebugMsg, QtFatalMsg,
|
||||
QtWarningMsg)
|
||||
|
||||
|
||||
CSConverter = None
|
||||
def coffee2js(script):
|
||||
global CSConverter
|
||||
if not CSConverter:
|
||||
from csconverter import CSConverter
|
||||
return CSConverter().convert(script)
|
||||
def debug(debug_type):
|
||||
def excepthook(type_, value, tb):
|
||||
import traceback
|
||||
|
||||
# print the exception...
|
||||
traceback.print_exception(type_, value, tb)
|
||||
print
|
||||
# ...then start the debugger in post-mortem mode
|
||||
pdb.pm()
|
||||
|
||||
def injectJsInFrame(filePath, scriptEncoding, libraryPath, targetFrame, startingScript=False):
|
||||
try:
|
||||
# if file doesn't exist in the CWD, use the lookup
|
||||
if not os.path.exists(filePath):
|
||||
filePath = os.path.join(libraryPath, filePath)
|
||||
# we are NOT in interactive mode
|
||||
if not hasattr(sys, 'ps1') or sys.stderr.target.isatty():
|
||||
import pdb
|
||||
|
||||
try:
|
||||
with codecs.open(filePath, encoding=scriptEncoding) as f:
|
||||
script = f.read()
|
||||
except UnicodeDecodeError as e:
|
||||
sys.exit("%s in '%s'" % (e, filePath))
|
||||
from PyQt4.QtCore import pyqtRemoveInputHook
|
||||
pyqtRemoveInputHook()
|
||||
|
||||
if script.startswith('#!') and not filePath.lower().endswith('.coffee'):
|
||||
script = '//' + script
|
||||
|
||||
if filePath.lower().endswith('.coffee'):
|
||||
result = coffee2js(script)
|
||||
if not result[0]:
|
||||
if startingScript:
|
||||
sys.exit("%s: '%s'" % (result[1], filePath))
|
||||
else:
|
||||
qDebug("%s: '%s'" % (result[1], filePath))
|
||||
script = ''
|
||||
else:
|
||||
script = result[1]
|
||||
|
||||
targetFrame.evaluateJavaScript(script)
|
||||
return True
|
||||
except IOError as (t, e):
|
||||
qDebug("%s: '%s'" % (e, filePath))
|
||||
return False
|
||||
if debug_type == 'exception':
|
||||
sys.excepthook = excepthook
|
||||
elif debug_type == 'program':
|
||||
pdb.set_trace()
|
||||
|
||||
|
||||
class MessageHandler(object):
|
||||
|
|
|
|||
|
|
@ -17,6 +17,9 @@
|
|||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
'''
|
||||
|
||||
import codecs
|
||||
import os
|
||||
import sys
|
||||
from cStringIO import StringIO
|
||||
from math import ceil, floor
|
||||
|
||||
|
|
@ -36,9 +39,42 @@ try:
|
|||
except ImportError:
|
||||
qDebug('PIL not found! Saving to gif files will be disabled.')
|
||||
|
||||
from csconverter import CSConverter
|
||||
from networkaccessmanager import NetworkAccessManager
|
||||
from plugincontroller import do_action
|
||||
from utils import injectJsInFrame
|
||||
|
||||
|
||||
def injectJsInFrame(filePath, scriptEncoding, libraryPath, targetFrame, startingScript=False):
|
||||
try:
|
||||
# if file doesn't exist in the CWD, use the lookup
|
||||
if not os.path.exists(filePath):
|
||||
filePath = os.path.join(libraryPath, filePath)
|
||||
|
||||
try:
|
||||
with codecs.open(filePath, encoding=scriptEncoding) as f:
|
||||
script = f.read()
|
||||
except UnicodeDecodeError as e:
|
||||
sys.exit("%s in '%s'" % (e, filePath))
|
||||
|
||||
if script.startswith('#!') and not filePath.lower().endswith('.coffee'):
|
||||
script = '//' + script
|
||||
|
||||
if filePath.lower().endswith('.coffee'):
|
||||
result = CSConverter().convert(script)
|
||||
if not result[0]:
|
||||
if startingScript:
|
||||
sys.exit("%s: '%s'" % (result[1], filePath))
|
||||
else:
|
||||
qDebug("%s: '%s'" % (result[1], filePath))
|
||||
script = ''
|
||||
else:
|
||||
script = result[1]
|
||||
|
||||
targetFrame.evaluateJavaScript(script)
|
||||
return True
|
||||
except IOError as (t, e):
|
||||
qDebug("%s: '%s'" % (e, filePath))
|
||||
return False
|
||||
|
||||
|
||||
class CustomPage(QWebPage):
|
||||
|
|
|
|||
Loading…
Reference in New Issue