readme updates

Add support for PPC64 Little Endian, fix incorrect spelling in wrp.go and README.md

.gitignore

@@ -1 +1 @@
-.vscode
+wrp-*

Makefile

@@ -0,0 +1,25 @@
+all: linux freebsd openbsd macos windows rpi
+
+linux: 
+	GOOS=linux GOARCH=amd64 go build -a -o wrp-linux wrp.go
+
+linux-ppc64le:
+	GOOS=linux GOARCH=ppc64le go build -a -o wrp-linux-ppc64le wrp.go
+
+rpi:
+	GOOS=linux GOARCH=arm go build -a -o wrp-linux-rpi wrp.go
+
+freebsd: 
+	GOOS=freebsd GOARCH=amd64 go build -a -o wrp-freebsd wrp.go
+
+openbsd: 
+	GOOS=openbsd GOARCH=amd64 go build -a -o wrp-openbsd wrp.go
+
+macos: 
+	GOOS=darwin GOARCH=amd64 go build -a -o wrp-macos wrp.go
+
+windows: 
+	GOOS=windows GOARCH=amd64 go build -a -o wrp-windows.exe wrp.go
+
+clean:
+	rm -rf wrp-linux wrp-freebsd wrp-openbsd wrp-macos wrp-windows.exe wrp-linux-rpi

README.md

@@ -1,32 +1,46 @@
 # WRP - Web Rendering Proxy
 
-A HTTP proxy server that allows to use historical and obsolete web browsers on the modern web. It works by rendering the web page in to a GIF image associated with clickable imagemap of original web links.
+A HTTP proxy server that allows to use historical and obsolete web browsers on the modern web. It works by rendering the web page in to a GIF image. It sends mouse clicks via ISMAP and keystrokes from a text box form input.
 
-**You are looking at a GoLang / CDP branch of WRP.**
+## Current Status
 
-**This code is under active development and not fully usable yet.**
+* This is a new reimplementation in GoLang/[ChromeDP](https://github.com/chromedp/chromedp). Python/Webkit being now deprecated.
+* Beta versiom, but fully supported an maintained.
+* Works as browser-in-browser. A real http proxy mode is being investigated. Check [issue #35](https://github.com/tenox7/wrp/issues/35) for updates.
+* As of 4.1 supports clicking on non-link elements (eg. cookie warnings, dropdown menus, etc.) and sending keystrokes. Yes, you can login and use Gmail or play web based games from any old browser.
 
-## Done so far
+## Usage	
 
-* basic browser-in-browser mode
-* screenshot and serve image+map via CDP
-* gif with Floyd–Steinberg dithering
-* random image addressing
-* resolve relative links
-* paginated scrolling
-* google search on input not starting with ^http
+1. [Download a WRP binary](https://github.com/tenox7/wrp/releases/) and run it on a machine that will become your WRP server.
+2. Point your legacy browser to `http://address:port` of WRP server. Do not set or use it as a "Proxy Server" (yet).
+3. Type a search string or a http/https URL and click GO.	
+4. Adjust your screen width/height/scale/#colors to fit in your old browser.	
+5. Scroll web page by clicking on the in-image scroll bar.
+6. Send keystrokes by filling in K input box and pressing Go. You also have buttons for backspace, enter and arrow keys.
 
-## Todo
+![Internet Explorer 1.5 doing Gmail](wrp.png)
 
-* ISMAP - underway
-* net/url: invalid control character in URL on Windows
-* configurable color palete and quantization
-* real http proxy support
-* option to encode as png/jpeg
-* padded box model coordinates
-* better http server shutdown
-* chromedp logging, timeout, non-headless flags
+## Flags
+```
+-l  listen address:port, default :8080
+-h  headed mode, display browser window
+-d  chromedp debug logging
+```
 
-## Python version
+## History
+* In 2014, version 1.0 started as a cgi-bin script, adaptation of `webkit2png.py` and `pcidade.py`, [blog post](https://virtuallyfun.com/2014/03/03/surfing-modern-web-with-ancient-browsers/).
+* Later in 2014, version 2.0 became a stand alone http-proxy server, also support for both Linux/MacOS, [another post](https://virtuallyfun.com/wordpress/2014/03/11/web-rendering-proxy-update//).
+* In 2016 the whole internet migrated to HTTPS/SSL/TLS and WRP largely stopped working. Python code became unmaintainable and mostly unportable (especially to Windows, even WSL).
+* In 2019 WRP 3.0 has been rewritten in Golang/Chromedp as browser-in-browser instead of http proxy.
+* Later in 2019, WRP 4.0 has been completely refactored to use mouse clicks instead using a href nodes. Also in 4.1 added sending keystrokes in to input boxes. You can now login to Gmail.
 
-Check [master branch](https://github.com/tenox7/wrp/tree/master) for "stable" Python-Webkit version.
+## Credits 
+* Uses [chromedp](https://github.com/chromedp), thanks to [mvdan](https://github.com/mvdan) for dealing with my issues
+* Uses [go-quantize](https://github.com/ericpauley/go-quantize), thanks to [ericpauley](https://github.com/ericpauley) for developing the missing go quantizer
+* Thanks to Jason Stevens of [Fun With Virtualization](https://virtuallyfun.com/) for graciously hosting my rumblings
+* Thanks to [claunia](https://github.com/claunia/) for help with the Python/Webkit version in the past
+
+## Legal Stuff
+License: Apache 2.0  
+Copyright (c) 2013-2018 Antoni Sawicki  
+Copyright (c) 2019 Google LLC

make.ps1

@@ -0,0 +1,20 @@
+param (
+    [switch]$clean = $false
+)
+$env:GOARCH="amd64"
+foreach($os in ("linux", "freebsd", "openbsd", "darwin", "windows")) {
+    $env:GOOS=$os
+    $o="wrp-$(if ($os -eq "windows") {$os="windows.exe"})$os"
+    Remove-Item -ErrorAction Ignore $o
+    if (!$clean) {
+        Invoke-Expression "& go build -a -o $o wrp.go"
+    }
+}
+
+$env:GOARCH="arm"
+$env:GOOS="linux"
+$o="wrp-linux-rpi"
+Remove-Item -ErrorAction Ignore  $o
+if (!$clean) {
+    Invoke-Expression "& go build -a -o $o wrp.go"
+}

old/README

@@ -0,0 +1,9 @@
+Historical versions of WRP and prior art
+
+License: GNU
+
+Copyright (c) 2013-2018 Antoni Sawicki
+Copyright (c) 2012-2013 picidae.net
+Copyright (c) 2004-2013 Paul Hammond
+Copyright (c) 2017-2018 Natalia Portillo
+Copyright (c) 2018      //gir.st/

old/picidae.py

@@ -0,0 +1,392 @@
+#!/usr/bin/env python
+
+# picidae.py - makes screenshots of webpages
+# and analyzes the webpage structure and writes image-maps of the links
+# as well as forms that are placed on the exact position of the old form.
+# It is a part of the art project www.picidae.net
+# http://www.picidae.net
+
+#
+# This script is based on webkit2png from Paul Hammond.
+# It was extended by picidae.net
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+
+
+__version__ = "1.0"
+
+
+import sys
+
+
+#print "hello ... "
+
+
+try:
+  import Foundation
+  import WebKit
+  import AppKit
+  import objc
+  import urllib
+except ImportError:
+  print "Cannot find pyobjc library files.  Are you sure it is installed?"
+  sys.exit()
+
+
+
+
+
+#try:
+#  from optparse import OptionParser
+#except ImportError:
+#  print "OptionParser not imported"
+#  sys.exit()
+
+from optparse import OptionParser
+
+
+class AppDelegate (Foundation.NSObject):
+    # what happens when the app starts up
+    def applicationDidFinishLaunching_(self, aNotification):
+        webview = aNotification.object().windows()[0].contentView()
+        webview.frameLoadDelegate().getURL(webview)
+
+
+class WebkitLoad (Foundation.NSObject, WebKit.protocols.WebFrameLoadDelegate):
+    # what happens if something goes wrong while loading
+    def webView_didFailLoadWithError_forFrame_(self,webview,error,frame):
+        print " ... something went wrong 1"
+        self.getURL(webview)
+    def webView_didFailProvisionalLoadWithError_forFrame_(self,webview,error,frame):
+        print " ... something went wrong 2"
+        self.getURL(webview)
+
+    def makeFilename(self,URL,options):
+       # make the filename
+       if options.filename:
+         filename = options.filename
+       elif options.md5:
+         try:
+                import md5
+         except ImportError:
+                print "--md5 requires python md5 library"
+                AppKit.NSApplication.sharedApplication().terminate_(None)
+         filename = md5.new(URL).hexdigest()
+       else:
+         import re
+         filename = re.sub('\W','',URL);
+         filename = re.sub('^http','',filename);
+       if options.datestamp:
+         import time
+         now = time.strftime("%Y%m%d")
+         filename = now + "-" + filename
+       import os
+       dir = os.path.abspath(os.path.expanduser(options.dir))
+       return os.path.join(dir,filename)
+
+    def saveImages(self,bitmapdata,filename,options):
+        # save the fullsize png
+        if options.fullsize:
+            bitmapdata.representationUsingType_properties_(AppKit.NSPNGFileType,None).writeToFile_atomically_(filename + ".png",objc.YES)
+
+        if options.thumb or options.clipped:
+            # work out how big the thumbnail is
+            width = bitmapdata.pixelsWide()
+            height = bitmapdata.pixelsHigh()
+            thumbWidth = (width * options.scale)
+            thumbHeight = (height * options.scale)
+
+            # make the thumbnails in a scratch image
+            scratch = AppKit.NSImage.alloc().initWithSize_(
+                    Foundation.NSMakeSize(thumbWidth,thumbHeight))
+            scratch.lockFocus()
+            AppKit.NSGraphicsContext.currentContext().setImageInterpolation_(
+                    AppKit.NSImageInterpolationHigh)
+            thumbRect = Foundation.NSMakeRect(0.0, 0.0, thumbWidth, thumbHeight)
+            clipRect = Foundation.NSMakeRect(0.0,
+                    thumbHeight-options.clipheight,
+                    options.clipwidth, options.clipheight)
+            bitmapdata.drawInRect_(thumbRect)
+            thumbOutput = AppKit.NSBitmapImageRep.alloc().initWithFocusedViewRect_(thumbRect)
+            clipOutput = AppKit.NSBitmapImageRep.alloc().initWithFocusedViewRect_(clipRect)
+            scratch.unlockFocus()
+           
+            # save the thumbnails as pngs
+            if options.thumb:
+                thumbOutput.representationUsingType_properties_(
+                        AppKit.NSPNGFileType,None
+                    ).writeToFile_atomically_(filename + "-thumb.png",objc.YES)
+            if options.clipped:
+                clipOutput.representationUsingType_properties_(
+                        AppKit.NSPNGFileType,None
+                    ).writeToFile_atomically_(filename + "-clipped.png",objc.YES)
+
+    def getURL(self,webview):
+        if self.urls:
+            if self.urls[0] == '-':
+                url = sys.stdin.readline().rstrip()
+                if not url: AppKit.NSApplication.sharedApplication().terminate_(None)
+            else:
+                url = self.urls.pop(0)
+        else:
+            AppKit.NSApplication.sharedApplication().terminate_(None)
+        #print "<urlcall href=\"\" />", url, "..."
+        #print "<urlcall href=\"%s\" />" % (url)
+        self.resetWebview(webview)
+        webview.mainFrame().loadRequest_(Foundation.NSURLRequest.requestWithURL_(Foundation.NSURL.URLWithString_(url)))
+        if not webview.mainFrame().provisionalDataSource():
+            print "<nosuccess  />"
+            self.getURL(webview)
+     
+    def resetWebview(self,webview):
+        rect = Foundation.NSMakeRect(0,0,self.options.initWidth,self.options.initHeight)
+        webview.window().setContentSize_((self.options.initWidth,self.options.initHeight))
+        webview.setFrame_(rect)
+   
+    def resizeWebview(self,view):
+        view.window().display()
+        view.window().setContentSize_(view.bounds().size)
+        view.setFrame_(view.bounds())
+
+    def captureView(self,view):
+        view.lockFocus()
+        bitmapdata = AppKit.NSBitmapImageRep.alloc()
+        bitmapdata.initWithFocusedViewRect_(view.bounds())
+        view.unlockFocus()
+        return bitmapdata
+ 
+    # what happens when the page has finished loading
+    def webView_didFinishLoadForFrame_(self,webview,frame):
+        # don't care about subframes
+        if (frame == webview.mainFrame()):
+            view = frame.frameView().documentView()
+
+            self.resizeWebview(view)
+
+            URL = frame.dataSource().initialRequest().URL().absoluteString()
+            filename = self.makeFilename(URL, self.options)
+
+            bitmapdata = self.captureView(view)  
+            self.saveImages(bitmapdata,filename,self.options)
+
+            # ----------------------------------
+            # picidae my stuff
+
+
+            #print "url"
+            print "<page>"
+            print frame.dataSource().request().URL().absoluteString()
+            print "</page>"
+
+
+            # Analyse HTML and get links
+            xmloutput = "<map name=\"map\">\r";
+           
+            domdocument = frame.DOMDocument()
+            domnodelist = domdocument.getElementsByTagName_('A')
+            i = 0
+            while  i < domnodelist.length():
+                # linkvalue
+                value = domnodelist.item_(i).valueForKey_('href')
+               
+                # position-rect
+                myrect = domnodelist.item_(i).boundingBox()
+               
+                xmin = Foundation.NSMinX(myrect)
+                ymin = Foundation.NSMinY(myrect)
+                xmax = Foundation.NSMaxX(myrect)
+                ymax = Foundation.NSMaxY(myrect)
+               
+                # print Link
+                prefix = ""
+                xmloutput += "<area shape=\"rect\" coords=\"%i,%i,%i,%i\" alt=\"\"><![CDATA[%s%s]]></area>\r" % (xmin, ymin, xmax, ymax, prefix, value)
+                i += 1
+           
+            #print "</map>"
+            xmloutput += "</map>"
+            f = open(filename +'.xml', 'w+')
+            f.write(xmloutput)
+            f.close()
+           
+            # ----------------------------------
+            # get forms
+            xmloutput = "<forms>\r";
+            xmloutput += "<page><![CDATA["
+            xmloutput += frame.dataSource().request().URL().absoluteString()
+            xmloutput += "]]></page>\r"
+           
+            domdocument = frame.DOMDocument()
+            domnodelist = domdocument.getElementsByTagName_('form')
+            i = 0
+            while  i < domnodelist.length():
+                # form
+                action = domnodelist.item_(i).valueForKey_('action')
+                method = domnodelist.item_(i).valueForKey_('method')
+                xmloutput += "<form method=\"%s\" ><action><![CDATA[%s]]></action>\r" % (method, action)
+               
+                # form fields
+                fieldlist = domnodelist.item_(i).getElementsByTagName_('input')
+                j=0
+                while  j < fieldlist.length():
+                        # values
+                        type = fieldlist.item_(j).valueForKey_('type')
+                        name = fieldlist.item_(j).valueForKey_('name')
+                        formvalue = fieldlist.item_(j).valueForKey_('value')
+                        size = fieldlist.item_(j).valueForKey_('size')
+                        checked = fieldlist.item_(j).valueForKey_('checked')
+                        # write output
+                        xmloutput += "\t<input "
+                        if (type):
+                                xmloutput += "type=\"%s\" " % (type)
+                        if (name):
+                                xmloutput += "name=\"%s\" " % (name)
+                        if (size):
+                                xmloutput += "size=\"%s\" " % (size)
+                        if (type and type != "hidden"):
+                                myrect = fieldlist.item_(j).boundingBox()
+                                xmin = Foundation.NSMinX(myrect)
+                                ymin = Foundation.NSMinY(myrect)
+                                xmax = Foundation.NSMaxX(myrect)
+                                ymax = Foundation.NSMaxY(myrect)
+                                height = ymax - ymin
+                                width = xmax - xmin
+                                if (type == "radio" or type == "checkbox"):
+                                        xmin -= 3
+                                        ymin -= 3
+                                xmloutput += "style=\"position:absolute;top:%i;left:%i;width:%i;height:%i;\" " % (ymin, xmin, width, height)
+                        if (checked):
+                                xmloutput += "checked=\"%s\" " % (checked)
+                        xmloutput += "><![CDATA["      
+                        if (formvalue and type!="text" and type!="password"):
+                                #xmloutput += urllib.quote(formvalue)
+                                dummy=10
+                        xmloutput += "]]></input>\r"
+                        j += 1
+                xmloutput += "</form>\r"
+                i += 1
+           
+            xmloutput += "</forms>"
+            f = open(filename +'.form.xml', 'w+')
+            f.write(xmloutput)
+            f.close()
+           
+           
+            # End picidae
+            # ----------------------------------
+           
+           
+            #print " ... done"
+            self.getURL(webview)
+           
+            #trying to give back the real url
+
+
+def main():
+       
+    # parse the command line
+    usage = """%prog [options] [http://example.net/ ...]
+
+examples:
+%prog http://google.com/            # screengrab google
+%prog -W 1000 -H 1000 http://google.com/ # bigger screengrab of google
+%prog -T http://google.com/         # just the thumbnail screengrab
+%prog -TF http://google.com/        # just thumbnail and fullsize grab
+%prog -o foo http://google.com/     # save images as "foo-thumb.png" etc
+%prog -                             # screengrab urls from stdin"""
+
+    cmdparser = OptionParser(usage, version=("webkit2png "+__version__))
+    # TODO: add quiet/verbose options
+    cmdparser.add_option("-W", "--width",type="float",default=800.0,
+       help="initial (and minimum) width of browser (default: 800)")
+    cmdparser.add_option("-H", "--height",type="float",default=600.0,
+       help="initial (and minimum) height of browser (default: 600)")
+    cmdparser.add_option("--clipwidth",type="float",default=200.0,
+       help="width of clipped thumbnail (default: 200)",
+       metavar="WIDTH")
+    cmdparser.add_option("--clipheight",type="float",default=150.0,
+       help="height of clipped thumbnail (default: 150)",
+       metavar="HEIGHT")
+    cmdparser.add_option("-s", "--scale",type="float",default=0.25,
+       help="scale factor for thumbnails (default: 0.25)")
+    cmdparser.add_option("-m", "--md5", action="store_true",
+       help="use md5 hash for filename (like del.icio.us)")
+    cmdparser.add_option("-o", "--filename", type="string",default="",
+       metavar="NAME", help="save images as NAME.png,NAME-thumb.png etc")
+    cmdparser.add_option("-F", "--fullsize", action="store_true",
+       help="only create fullsize screenshot")
+    cmdparser.add_option("-T", "--thumb", action="store_true",
+       help="only create thumbnail sreenshot")
+    cmdparser.add_option("-C", "--clipped", action="store_true",
+       help="only create clipped thumbnail screenshot")
+    cmdparser.add_option("-d", "--datestamp", action="store_true",
+       help="include date in filename")
+    cmdparser.add_option("-D", "--dir",type="string",default="./",
+       help="directory to place images into")
+    (options, args) = cmdparser.parse_args()
+    if len(args) == 0:
+        cmdparser.print_help()
+        return
+    if options.filename:
+        if len(args) != 1 or args[0] == "-":
+          print "--filename option requires exactly one url"
+          return
+    if options.scale == 0:
+      cmdparser.error("scale cannot be zero")
+    # make sure we're outputing something
+    if not (options.fullsize or options.thumb or options.clipped):
+      options.fullsize = True
+      options.thumb = True
+      options.clipped = True
+    # work out the initial size of the browser window
+    #  (this might need to be larger so clipped image is right size)
+    options.initWidth = (options.clipwidth / options.scale)
+    options.initHeight = (options.clipheight / options.scale)
+    if options.width>options.initWidth:
+       options.initWidth = options.width
+    if options.height>options.initHeight:
+       options.initHeight = options.height
+   
+   
+    app = AppKit.NSApplication.sharedApplication()
+   
+    # create an app delegate
+    delegate = AppDelegate.alloc().init()
+    AppKit.NSApp().setDelegate_(delegate)
+       
+    # create a window
+    rect = Foundation.NSMakeRect(-16000,-16000,100,100)
+    win = AppKit.NSWindow.alloc()
+    win.initWithContentRect_styleMask_backing_defer_ (rect,
+            AppKit.NSBorderlessWindowMask, 2, 0)
+       
+    # create a webview object
+    webview = WebKit.WebView.alloc()
+    webview.initWithFrame_(rect)
+    # turn off scrolling so the content is actually x wide and not x-15
+    webview.mainFrame().frameView().setAllowsScrolling_(objc.NO)
+    # add the webview to the window
+    win.setContentView_(webview)
+   
+   
+    # create a LoadDelegate
+    loaddelegate = WebkitLoad.alloc().init()
+    loaddelegate.options = options
+    loaddelegate.urls = args
+    webview.setFrameLoadDelegate_(loaddelegate)
+       
+    app.run()    
+
+if __name__ == '__main__' : main()
+

old/webkit2png.py

@@ -0,0 +1,506 @@
+#!/usr/bin/python
+
+# webkit2png - makes screenshots of web pages
+# http://www.paulhammond.org/webkit2png
+
+__version__ = "dev"
+
+# Copyright (c) 2004-2013 Paul Hammond
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import sys
+import optparse
+import re
+import os
+
+try:
+    import Foundation
+    import WebKit
+    import AppKit
+    import Quartz
+    import objc
+except ImportError:
+    print "Cannot find pyobjc library files.  Are you sure it is installed?"
+    sys.exit()
+
+
+class AppDelegate(Foundation.NSObject):
+    # what happens when the app starts up
+    def applicationDidFinishLaunching_(self, aNotification):
+        webview = aNotification.object().windows()[0].contentView()
+        webview.frameLoadDelegate().getURL(webview)
+        self.performSelector_withObject_afterDelay_("timeout:", None,
+                                                    self.timeout)
+
+    def timeout_(self, obj):
+        Foundation.NSLog("timed out!")
+        AppKit.NSApplication.sharedApplication().terminate_(None)
+
+
+class Webkit2PngScriptBridge(Foundation.NSObject):
+    def init(self):
+        self = super(Webkit2PngScriptBridge, self).init()
+        self.is_stopped = False
+        self.start_callback = False
+        return self
+
+    def stop(self):
+        self.is_stopped = True
+
+    def start(self):
+        self.is_stopped = False
+        self.start_callback()
+
+    def isSelectorExcludedFromWebScript_(self, sel):
+        if sel in ['stop', 'start']:
+            return False
+        else:
+            return True
+
+
+class WebkitLoad (Foundation.NSObject, WebKit.protocols.WebFrameLoadDelegate):
+
+    # what happens if something goes wrong while loading
+    def webView_didFailLoadWithError_forFrame_(self, webview, error, frame):
+        if error.code() == Foundation.NSURLErrorCancelled:
+            return
+        print " ... something went wrong: "+error.localizedDescription()
+        self.getURL(webview)
+
+    def webView_didFailProvisionalLoadWithError_forFrame_(self, webview, error,
+                                                          frame):
+        if error.code() == Foundation.NSURLErrorCancelled:
+            return
+        print " ... something went wrong: "+error.localizedDescription()
+        self.getURL(webview)
+
+    def makeFilename(self, URL, options):
+        # make the filename
+        if options.filename:
+            filename = options.filename
+        elif options.md5:
+            try:
+                import md5
+            except ImportError:
+                print "--md5 requires python md5 library"
+                AppKit.NSApplication.sharedApplication().terminate_(None)
+            filename = md5.new(URL).hexdigest()
+        else:
+            filename = re.sub('^https?', '', URL)
+            filename = re.sub('\W', '', filename)
+        if options.datestamp:
+            import time
+            now = time.strftime("%Y%m%d")
+            filename = now + "-" + filename
+        dir = os.path.abspath(os.path.expanduser(options.dir))
+        if not os.path.exists(options.dir):
+            os.makedirs(dir)
+        return os.path.join(dir, filename)
+
+    def saveImages(self, bitmapdata, filename, options):
+        # save the fullsize png
+        if options.fullsize:
+            bitmapdata.representationUsingType_properties_(
+                AppKit.NSPNGFileType,
+                None
+            ).writeToFile_atomically_(filename + "-full.png", objc.YES)
+
+        if options.thumb or options.clipped:
+            # work out how big the thumbnail is
+            width = bitmapdata.pixelsWide()
+            height = bitmapdata.pixelsHigh()
+            thumbWidth = (width * options.scale)
+            thumbHeight = (height * options.scale)
+
+            # make the thumbnails in a scratch image
+            scratch = AppKit.NSImage.alloc().initWithSize_(
+                Foundation.NSMakeSize(thumbWidth, thumbHeight))
+            scratch.lockFocus()
+            AppKit.NSGraphicsContext.currentContext().setImageInterpolation_(
+                AppKit.NSImageInterpolationHigh)
+            thumbRect = Foundation.NSMakeRect(0.0, 0.0, thumbWidth,
+                                              thumbHeight)
+            clipRect = Foundation.NSMakeRect(
+                0.0,
+                thumbHeight-options.clipheight,
+                options.clipwidth,
+                options.clipheight)
+            bitmapdata.drawInRect_(thumbRect)
+            thumbOutput = AppKit.NSBitmapImageRep.alloc()\
+                .initWithFocusedViewRect_(thumbRect)
+            clipOutput = AppKit.NSBitmapImageRep.alloc()\
+                .initWithFocusedViewRect_(clipRect)
+            scratch.unlockFocus()
+
+            # save the thumbnails as pngs
+            if options.thumb:
+                thumbOutput.representationUsingType_properties_(
+                    AppKit.NSPNGFileType, None).writeToFile_atomically_(
+                        filename + "-thumb.png", objc.YES)
+            if options.clipped:
+                clipOutput.representationUsingType_properties_(
+                    AppKit.NSPNGFileType, None).writeToFile_atomically_(
+                        filename + "-clipped.png", objc.YES)
+
+    def getURL(self, webview):
+        if self.urls:
+            if self.urls[0] == '-':
+                url = sys.stdin.readline().rstrip()
+                if not url:
+                    AppKit.NSApplication.sharedApplication().terminate_(None)
+            else:
+                url = self.urls.pop(0)
+        else:
+            AppKit.NSApplication.sharedApplication().terminate_(None)
+
+        nsurl = Foundation.NSURL.URLWithString_(url)
+        if not (nsurl and nsurl.scheme()):
+                nsurl = Foundation.NSURL.alloc().initFileURLWithPath_(url)
+        nsurl = nsurl.absoluteURL()
+
+        if self.options.ignore_ssl_check:
+            Foundation.NSURLRequest.setAllowsAnyHTTPSCertificate_forHost_(objc.YES, nsurl.host())
+
+        print "Fetching", nsurl, "..."
+        self.resetWebview(webview)
+        scriptobject = webview.windowScriptObject()
+        scriptobject.setValue_forKey_(Webkit2PngScriptBridge.alloc().init(),
+                                      'webkit2png')
+
+        webview.mainFrame().loadRequest_(Foundation.NSURLRequest.requestWithURL_(nsurl))
+        if not webview.mainFrame().provisionalDataSource():
+            print " ... not a proper url?"
+            self.getURL(webview)
+
+    def resetWebview(self, webview):
+        rect = Foundation.NSMakeRect(0, 0, self.options.initWidth,
+                                     self.options.initHeight)
+        window = webview.window()
+        window.setContentSize_((self.options.initWidth,
+                                self.options.initHeight))
+
+        if self.options.transparent:
+            window.setOpaque_(objc.NO)
+            window.setBackgroundColor_(AppKit.NSColor.clearColor())
+            webview.setDrawsBackground_(objc.NO)
+
+        webview.setFrame_(rect)
+
+    def captureView(self, view):
+        bounds = view.bounds()
+        if bounds.size.height > self.options.UNSAFE_max_height:
+            print >> sys.stderr, "Error: page height greater than %s, " \
+                "clipping to avoid crashing windowserver." % \
+                self.options.UNSAFE_max_height
+            bounds.size.height = self.options.UNSAFE_max_height
+        if bounds.size.width > self.options.UNSAFE_max_width:
+            print >> sys.stderr, "Error: page width greater than %s, " \
+                "clipping to avoid crashing windowserver." % \
+                self.options.UNSAFE_max_width
+            bounds.size.width = self.options.UNSAFE_max_width
+
+        view.window().display()
+        view.window().setContentSize_(
+            Foundation.NSSize(self.options.initWidth, self.options.initHeight))
+        view.setFrame_(bounds)
+
+        if hasattr(view, "bitmapImageRepForCachingDisplayInRect_"):
+            bitmapdata = view.bitmapImageRepForCachingDisplayInRect_(bounds)
+            view.cacheDisplayInRect_toBitmapImageRep_(bounds, bitmapdata)
+        else:
+            view.lockFocus()
+            bitmapdata = AppKit.NSBitmapImageRep.alloc()
+            bitmapdata.initWithFocusedViewRect_(bounds)
+            view.unlockFocus()
+        return bitmapdata
+
+    # what happens when the page has finished loading
+    def webView_didFinishLoadForFrame_(self, webview, frame):
+        # don't care about subframes
+        if (frame == webview.mainFrame()):
+            scriptobject = webview.windowScriptObject()
+            if self.options.js:
+                scriptobject.evaluateWebScript_(self.options.js)
+
+            bridge = scriptobject.valueForKey_('webkit2png')
+
+            def doGrab():
+                Foundation.NSTimer.\
+                    scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(
+                        self.options.delay, self, self.doGrab, webview, False)
+
+            if bridge.is_stopped:
+                bridge.start_callback = doGrab
+            else:
+                doGrab()
+
+    def doGrab(self, timer):
+        webview = timer.userInfo()
+        frame = webview.mainFrame()
+        view = frame.frameView().documentView()
+
+        URL = webview.mainFrame().dataSource().initialRequest().URL()\
+            .absoluteString()
+        filename = self.makeFilename(URL, self.options)
+
+        bitmapdata = self.captureView(view)
+
+        if self.options.selector:
+            doc = frame.DOMDocument()
+            el = doc.querySelector_(self.options.selector)
+
+            if not el:
+                print " ... no element matching %s found?" % \
+                    self.options.selector
+                self.getURL(webview)
+                return
+
+            left, top = 0, 0
+            parent = el
+            while parent:
+                left += parent.offsetLeft()
+                top += parent.offsetTop()
+                parent = parent.offsetParent()
+
+            zoom = self.options.zoom
+
+            cropRect = view.window().convertRectToBacking_(Foundation.NSMakeRect(
+                zoom * left, zoom * top,
+                zoom * el.offsetWidth(), zoom * el.offsetHeight()))
+
+            cropped = Quartz.CGImageCreateWithImageInRect(
+                bitmapdata.CGImage(), cropRect)
+            bitmapdata = AppKit.NSBitmapImageRep.alloc().initWithCGImage_(
+                cropped)
+            Quartz.CGImageRelease(cropped)
+
+        self.saveImages(bitmapdata, filename, self.options)
+
+        print " ... done"
+        self.getURL(webview)
+
+
+def main():
+
+    # parse the command line
+    usage = """%prog [options] [http://example.net/ ...]
+
+Examples:
+%prog http://google.com/            # screengrab google
+%prog -W 1000 -H 1000 http://google.com/ # bigger screengrab of google
+%prog -T http://google.com/         # just the thumbnail screengrab
+%prog -TF http://google.com/        # just thumbnail and fullsize grab
+%prog -o foo http://google.com/     # save images as "foo-thumb.png" etc
+%prog -                             # screengrab urls from stdin
+%prog /path/to/file.html            # screengrab local html file
+%prog -h | less                     # full documentation"""
+
+    cmdparser = optparse.OptionParser(usage,
+                                      version=("webkit2png " + __version__))
+    # TODO: add quiet/verbose options
+    cmdparser.add_option("--debug", action="store_true",
+                         help=optparse.SUPPRESS_HELP)
+
+    # warning: setting these too high can crash your window server
+    cmdparser.add_option("--UNSAFE-max-height", type="int", default=30000,
+                         help=optparse.SUPPRESS_HELP)
+    cmdparser.add_option("--UNSAFE-max-width", type="int", default=30000,
+                         help=optparse.SUPPRESS_HELP)
+
+    group = optparse.OptionGroup(cmdparser, "Network Options")
+    group.add_option("--timeout", type="float", default=60.0,
+                     help="page load timeout (default: 60)")
+    group.add_option("--user-agent", type="string", default=False,
+                     help="set user agent header")
+    group.add_option("--ignore-ssl-check", action="store_true", default=False,
+                     help="ignore SSL Certificate name mismatches")
+    cmdparser.add_option_group(group)
+
+    group = optparse.OptionGroup(cmdparser, "Browser Window Options")
+    group.add_option(
+        "-W", "--width", type="float", default=800.0,
+        help="initial (and minimum) width of browser (default: 800)")
+    group.add_option(
+        "-H", "--height", type="float", default=600.0,
+        help="initial (and minimum) height of browser (default: 600)")
+    group.add_option(
+        "-z", "--zoom", type="float", default=1.0,
+        help='zoom level of browser, equivalent to "Zoom In" and "Zoom Out" '
+        'in "View" menu (default: 1.0)')
+    group.add_option(
+        "--selector", type="string",
+        help="CSS selector for a single element to capture (first matching "
+        "element will be used)")
+    cmdparser.add_option_group(group)
+
+    group = optparse.OptionGroup(cmdparser, "Output size options")
+    group.add_option(
+        "-F", "--fullsize", action="store_true",
+        help="only create fullsize screenshot")
+    group.add_option(
+        "-T", "--thumb", action="store_true",
+        help="only create thumbnail sreenshot")
+    group.add_option(
+        "-C", "--clipped", action="store_true",
+        help="only create clipped thumbnail screenshot")
+    group.add_option(
+        "--clipwidth", type="float", default=200.0,
+        help="width of clipped thumbnail (default: 200)",
+        metavar="WIDTH")
+    group.add_option(
+        "--clipheight", type="float", default=150.0,
+        help="height of clipped thumbnail (default: 150)",
+        metavar="HEIGHT")
+    group.add_option(
+        "-s", "--scale", type="float", default=0.25,
+        help="scale factor for thumbnails (default: 0.25)")
+    cmdparser.add_option_group(group)
+
+    group = optparse.OptionGroup(cmdparser, "Output filename options")
+    group.add_option(
+        "-D", "--dir", type="string", default="./",
+        help="directory to place images into")
+    group.add_option(
+        "-o", "--filename", type="string", default="",
+        metavar="NAME", help="save images as NAME-full.png,NAME-thumb.png etc")
+    group.add_option(
+        "-m", "--md5", action="store_true",
+        help="use md5 hash for filename (like del.icio.us)")
+    group.add_option(
+        "-d", "--datestamp", action="store_true",
+        help="include date in filename")
+    cmdparser.add_option_group(group)
+
+    group = optparse.OptionGroup(cmdparser, "Web page functionality")
+    group.add_option(
+        "--delay", type="float", default=0,
+        help="delay between page load finishing and screenshot")
+    group.add_option(
+        "--js", type="string", default=None,
+        help="JavaScript to execute when the window finishes loading"
+        "(example: --js='document.bgColor=\"red\";'). "
+        "If you need to wait for asynchronous code to finish before "
+        "capturing the screenshot, call webkit2png.stop() before the "
+        "async code runs, then webkit2png.start() to capture the image.")
+    group.add_option(
+        "--noimages", action="store_true",
+        help=optparse.SUPPRESS_HELP)
+    group.add_option(
+        "--no-images", action="store_true",
+        help="don't load images")
+    group.add_option(
+        "--nojs", action="store_true",
+        help=optparse.SUPPRESS_HELP)
+    group.add_option(
+        "--no-js", action="store_true",
+        help="disable JavaScript support")
+    group.add_option(
+        "--transparent", action="store_true",
+        help="render output on a transparent background (requires a web "
+        "page with a transparent background)", default=False)
+    cmdparser.add_option_group(group)
+
+    (options, args) = cmdparser.parse_args()
+    if len(args) == 0:
+        cmdparser.print_usage()
+        return
+    if options.filename:
+        if len(args) != 1 or args[0] == "-":
+            print "--filename option requires exactly one url"
+            return
+
+    # deprecated options
+    if options.nojs:
+        print >> sys.stderr, 'Warning: --nojs will be removed in ' \
+            'webkit2png 1.0. Please use --no-js.'
+        options.no_js = True
+    if options.noimages:
+        print >> sys.stderr, 'Warning: --noimages will be removed in ' \
+            'webkit2png 1.0. Please use --no-images.'
+        options.no_images = True
+
+    if options.scale == 0:
+        cmdparser.error("scale cannot be zero")
+    # make sure we're outputing something
+    if not (options.fullsize or options.thumb or options.clipped):
+        options.fullsize = True
+        options.thumb = True
+        options.clipped = True
+    # work out the initial size of the browser window
+    #  (this might need to be larger so clipped image is right size)
+    options.initWidth = (options.clipwidth / options.scale)
+    options.initHeight = (options.clipheight / options.scale)
+    options.width *= options.zoom
+    if options.width > options.initWidth:
+        options.initWidth = options.width
+    if options.height > options.initHeight:
+        options.initHeight = options.height
+
+    # Hide the dock icon (needs to run before NSApplication.sharedApplication)
+    AppKit.NSBundle.mainBundle().infoDictionary()['LSBackgroundOnly'] = '1'
+
+    app = AppKit.NSApplication.sharedApplication()
+
+    # create an app delegate
+    delegate = AppDelegate.alloc().init()
+    delegate.timeout = options.timeout
+    AppKit.NSApp().setDelegate_(delegate)
+
+    # create a window
+    rect = Foundation.NSMakeRect(0, 0, 100, 100)
+    win = AppKit.NSWindow.alloc()
+    win.initWithContentRect_styleMask_backing_defer_(
+        rect, AppKit.NSBorderlessWindowMask, 2, 0)
+    if options.debug:
+        win.orderFrontRegardless()
+    # create a webview object
+    webview = WebKit.WebView.alloc()
+    webview.initWithFrame_(rect)
+    # turn off scrolling so the content is actually x wide and not x-15
+    webview.mainFrame().frameView().setAllowsScrolling_(objc.NO)
+
+    if options.user_agent:
+        webview.setCustomUserAgent_(options.user_agent)
+    else:
+        webkit_version = Foundation.NSBundle.bundleForClass_(WebKit.WebView)\
+            .objectForInfoDictionaryKey_(WebKit.kCFBundleVersionKey)[1:]
+        webview.setApplicationNameForUserAgent_(
+            "Like-Version/6.0 Safari/%s webkit2png/%s" % (webkit_version, __version__))
+    webview.setPreferencesIdentifier_('webkit2png')
+    webview.preferences().setLoadsImagesAutomatically_(not options.no_images)
+    webview.preferences().setJavaScriptEnabled_(not options.no_js)
+
+    if options.zoom != 1.0:
+        webview._setZoomMultiplier_isTextOnly_(options.zoom, False)
+
+    # add the webview to the window
+    win.setContentView_(webview)
+
+    # create a LoadDelegate
+    loaddelegate = WebkitLoad.alloc().init()
+    loaddelegate.options = options
+    loaddelegate.urls = args
+    webview.setFrameLoadDelegate_(loaddelegate)
+
+    app.run()
+
+if __name__ == '__main__':
+    main()

old/webrender.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+
+# webrender.py - recursively render web pages to a gif+imagemap of clickable links
+# caveat: this script requires to be run as a regular user and cannot run as a daemon
+# from apache cgi-bin, you can use python built in http server instead
+# usage:
+#   create cgi-bin directory, copy webrender.py to cgi-bin and chmod 755
+#   python -m CGIHTTPServer 8000 
+#   navigate web browser to http://x.x.x.x:8000/cgi-bin/webrender.py
+# the webrender-xxx.gif images are created in the CWD of the http server
+
+
+__version__ = "1.0"
+
+# 
+# This program is based on the software picidae.py 1.0 from http://www.picidae.net
+# It was modified by Antoni Sawicki
+# 
+# This program is based on the software webkit2png 0.4 from Paul Hammond.
+# It was extended by picidae.net
+# 
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+                
+try:
+  import sys
+  import os
+  import glob
+  import random
+  import Foundation
+  import WebKit
+  import AppKit
+  import objc
+  import string
+  import urllib
+  import socket
+  import cgi
+  import cgitb; cgitb.enable() # for trubleshooting
+except ImportError:
+  print "Cannot find pyobjc library files.  Are you sure it is installed?"
+  sys.exit() 
+
+
+from optparse import OptionParser
+
+
+class AppDelegate (Foundation.NSObject):
+    # what happens when the app starts up
+    def applicationDidFinishLaunching_(self, aNotification):
+        webview = aNotification.object().windows()[0].contentView()
+        webview.frameLoadDelegate().getURL(webview)
+
+
+class WebkitLoad (Foundation.NSObject, WebKit.protocols.WebFrameLoadDelegate):
+    # what happens if something goes wrong while loading
+    def webView_didFailLoadWithError_forFrame_(self,webview,error,frame):
+        print " ... something went wrong 1: " + error.localizedDescription()
+        self.getURL(webview)
+    def webView_didFailProvisionalLoadWithError_forFrame_(self,webview,error,frame):
+        print " ... something went wrong 2: " + error.localizedDescription()
+        self.getURL(webview)
+
+    def getURL(self,webview):
+        if self.urls:
+            if self.urls[0] == '-':
+                url = sys.stdin.readline().rstrip()
+                if not url: AppKit.NSApplication.sharedApplication().terminate_(None)
+            else: 
+                url = self.urls.pop(0)
+        else:
+            AppKit.NSApplication.sharedApplication().terminate_(None)
+
+        self.resetWebview(webview)
+        webview.mainFrame().loadRequest_(Foundation.NSURLRequest.requestWithURL_(Foundation.NSURL.URLWithString_(url)))
+        if not webview.mainFrame().provisionalDataSource():
+            print "<nosuccess  />"
+            self.getURL(webview)
+     
+    def resetWebview(self,webview):
+        rect = Foundation.NSMakeRect(0,0,1024,768)
+        webview.window().setContentSize_((1024,768))
+        webview.setFrame_(rect)
+    
+    def resizeWebview(self,view):
+        view.window().display()
+        view.window().setContentSize_(view.bounds().size)
+        view.setFrame_(view.bounds())
+
+    def captureView(self,view):
+        view.lockFocus()
+        bitmapdata = AppKit.NSBitmapImageRep.alloc()
+        bitmapdata.initWithFocusedViewRect_(view.bounds())
+        view.unlockFocus()
+        return bitmapdata
+ 
+    # what happens when the page has finished loading
+    def webView_didFinishLoadForFrame_(self,webview,frame):
+        # don't care about subframes
+        if (frame == webview.mainFrame()):
+            view = frame.frameView().documentView()
+
+            self.resizeWebview(view)
+
+            URL = frame.dataSource().initialRequest().URL().absoluteString()
+
+            for fl in glob.glob("webrender-*.gif"):
+                os.remove(fl)
+
+            GIF = "webrender-%s.gif" % (random.randrange(0,1000))
+
+            bitmapdata = self.captureView(view)  
+            bitmapdata.representationUsingType_properties_(AppKit.NSGIFFileType,None).writeToFile_atomically_(GIF,objc.YES)
+
+            myurl = "http://%s:%s%s" % (socket.gethostbyname(socket.gethostname()), os.getenv("SERVER_PORT"), os.getenv("SCRIPT_NAME"))
+
+            print "Content-type: text/html\r\n\r\n"
+            print "<!-- webrender.py by Antoni Sawicki -->"
+            print "<html><head><title>Webrender - %s</title></head><body><table border=\"0\"><tr>" % (URL)
+            print "<td><form action=\"%s\">" % (myurl)
+            print "<input type=\"text\" name=\"url\" value=\"%s\" size=\"80\">" % (URL)
+            print "<input type=\"submit\" value=\"go\">"
+            print "</form></td><td>"
+            print "<form action=\"%s\">" % (myurl)
+            print "<input type=\"text\" name=\"search\" value=\"\" size=\"20\">"
+            print "<input type=\"submit\" value=\"search\">"
+            print "</form></td></tr></table>"
+            print "<img src=\"../%s\" alt=\"webrender\" usemap=\"#map\" border=\"0\">" % (GIF)
+
+
+            # Analyse HTML and get links
+            print "<map name=\"map\">";
+            
+            domdocument = frame.DOMDocument()
+            domnodelist = domdocument.getElementsByTagName_('A')
+            i = 0
+            while  i < domnodelist.length():
+            	# linkvalue
+            	value = domnodelist.item_(i).valueForKey_('href')
+            	
+            	# position-rect
+            	myrect = domnodelist.item_(i).boundingBox()
+            	
+            	xmin = Foundation.NSMinX(myrect)
+            	ymin = Foundation.NSMinY(myrect)
+            	xmax = Foundation.NSMaxX(myrect)
+            	ymax = Foundation.NSMaxY(myrect)
+            	
+            	# print Link
+            	escval = string.replace( string.replace(value, "?", "TNXQUE"), "&", "TNXAMP" )
+            	print "<area shape=\"rect\" coords=\"%i,%i,%i,%i\" alt=\"\" href=\"%s?url=%s\"></area>" % (xmin, ymin, xmax, ymax, myurl, escval)
+            	i += 1
+            
+            print "</map>"
+            print "</body></html>"
+            self.getURL(webview)
+
+
+def main():
+
+    # obtain url from cgi input
+    form = cgi.FieldStorage()
+    rawurl = form.getfirst("url", "http://www.google.com")
+    rawsearch = form.getfirst("search")
+    if rawsearch:
+        url = "http://www.google.com/search?q=%s" % (rawsearch)
+    else:
+        url = string.replace( string.replace(rawurl, "TNXAMP", "&"), "TNXQUE", "?")
+
+
+    AppKit.NSApplicationLoad();	
+
+    app = AppKit.NSApplication.sharedApplication()
+    
+    # create an app delegate
+    delegate = AppDelegate.alloc().init()
+    AppKit.NSApp().setDelegate_(delegate)
+	
+    # create a window
+    rect = Foundation.NSMakeRect(-16000,-16000,100,100)
+    win = AppKit.NSWindow.alloc()
+    win.initWithContentRect_styleMask_backing_defer_ (rect, AppKit.NSBorderlessWindowMask, 2, 0)
+	
+    # create a webview object
+    webview = WebKit.WebView.alloc()
+    webview.initWithFrame_(rect)
+    # turn off scrolling so the content is actually x wide and not x-15
+    webview.mainFrame().frameView().setAllowsScrolling_(objc.NO)
+    # add the webview to the window
+    win.setContentView_(webview)
+    
+   
+    # create a LoadDelegate
+    loaddelegate = WebkitLoad.alloc().init()
+    loaddelegate.options = [""]
+    loaddelegate.urls = [url]
+    webview.setFrameLoadDelegate_(loaddelegate)
+   	
+    app.run()    
+
+if __name__ == '__main__' : main()
+

old/wrp.py

@@ -284,10 +284,10 @@ if sys.platform.startswith('linux') or sys.platform.startswith('freebsd'):
                 if ISMAP == True:
                     mapfile.write("rect %s %i,%i %i,%i\n".decode('utf-8', errors='ignore') % (turl, xmin, ymin, xmax, ymax))
                 else:
-                    httpout.write("<AREA SHAPE=\"RECT\""
+                    httpout.write(("<AREA SHAPE=\"RECT\""
                                   " COORDS=\"%i,%i,%i,%i\""
                                   " ALT=\"%s\" HREF=\"%s\">\n".decode('utf-8', errors='ignore')
-                                  % (xmin, ymin, xmax, ymax, turl, turl))
+                                  % (xmin, ymin, xmax, ymax, turl, turl)).encode("utf-8"))
 
             if ISMAP != True:
                 httpout.write("</MAP>\n")

pywebkit/Changelog.md

@@ -1,36 +0,0 @@
-## [2.0] - 2017-05-10
-### Added
-- Support PyQt5 if available.
-- Sets title from original one.
-- Returns server errors as is.
-- Download non-HTML files as is.
-- For JavaScript capable browsers detect and automatically set view width.
-- Add support for configuring which image format to use.
-- Added support for PythonMagick. If found, allows to dither, color-reduce, or convert to grayscale or monochrome.
-- If PythonMagick is found, render as PNG and convert to user-requested format using it, for better quality.
-
-### Changed
-- Support www prepented to http://wrp.stop command.
-
-### Fixed
-- Prevent python crashes with non-ASCII character in URLs.
-
-## [1.4] - 2017-01-22
-### Added
-- Suport for ISMAP on Linux.
-- Use queues instead of globals in Linux.
-
-## [1.3] - 2017-01-21
-### Changed
-- Merged mac OS and Linux in a single executable.
-- Use queues instead of globals in Linux.
-
-### Fixed
-- Call PyQt to close application on http://wrp.stop
-
-## [1.2] - 2016-12-27
-### Added
-- Support for IMAP on mac OS.
-
-### Changed
-- Use queues instead of globals in mac OS.

wrp.go

@@ -12,7 +12,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	_ "image"
 	"image/gif"
 	"image/png"
 	"log"
@@ -20,200 +19,299 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"os/signal"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/chromedp/cdproto/emulation"
-	"github.com/chromedp/cdproto/runtime"
-
-	"github.com/chromedp/cdproto/cdp"
-	"github.com/chromedp/cdproto/dom"
 	"github.com/chromedp/chromedp"
+	"github.com/ericpauley/go-quantize/quantize"
 )
 
 var (
-	ctx    context.Context
-	cancel context.CancelFunc
-	gifmap = make(map[string]bytes.Buffer)
+	version = "4.2"
+	srv     http.Server
+	ctx     context.Context
+	cancel  context.CancelFunc
+	gifmap  = make(map[string]bytes.Buffer)
+	ismap   = make(map[string]wrpReq)
 )
 
-func pageServer(out http.ResponseWriter, r *http.Request) {
-	r.ParseForm()
-	u := r.FormValue("url")
-	var istr string
-	var i bool
-	if r.FormValue("i") == "on" {
-		istr = "CHECKED"
-		i = true
-	} else {
-		istr = ""
-		i = false
+type wrpReq struct {
+	U string  // url
+	W int64   // width
+	H int64   // height
+	S float64 // scale
+	C int64   // #colors
+	X int64   // mouseX
+	Y int64   // mouseY
+	K string  // keys to send
+	F string  // Fn buttons
+}
+
+func (w *wrpReq) parseForm(req *http.Request) {
+	req.ParseForm()
+	w.U = req.FormValue("url")
+	if len(w.U) > 1 && !strings.HasPrefix(w.U, "http") {
+		w.U = fmt.Sprintf("http://www.google.com/search?q=%s", url.QueryEscape(w.U))
 	}
-	p, _ := strconv.ParseInt(r.FormValue("p"), 10, 64)
-	if r.FormValue("pg") == ">>" {
-		p++
-	} else if r.FormValue("pg") == "<<" {
-		p--
-	} else {
-		p = 0
+	w.W, _ = strconv.ParseInt(req.FormValue("w"), 10, 64)
+	if w.W < 10 {
+		w.W = 1152
 	}
-	w, _ := strconv.ParseInt(r.FormValue("w"), 10, 64)
-	if w < 10 {
-		w = 1024
+	w.H, _ = strconv.ParseInt(req.FormValue("h"), 10, 64)
+	if w.H < 10 {
+		w.H = 600
 	}
-	h, _ := strconv.ParseInt(r.FormValue("h"), 10, 64)
-	if h < 10 {
-		h = 768
+	w.S, _ = strconv.ParseFloat(req.FormValue("s"), 64)
+	if w.S < 0.1 {
+		w.S = 1.0
 	}
-	s, _ := strconv.ParseFloat(r.FormValue("s"), 64)
-	if s < 0.1 {
-		s = 1.0
+	w.C, _ = strconv.ParseInt(req.FormValue("c"), 10, 64)
+	if w.C < 2 || w.C > 256 {
+		w.C = 256
 	}
-	log.Printf("%s Page Reqest for url=\"%s\" [%s]\n", r.RemoteAddr, u, r.URL.Path)
+	w.K = req.FormValue("k")
+	w.F = req.FormValue("Fn")
+	log.Printf("%s WrpReq from Form: %+v\n", req.RemoteAddr, w)
+}
+
+func (w wrpReq) printPage(out http.ResponseWriter) {
+	out.Header().Set("Cache-Control", "max-age=0")
+	out.Header().Set("Expires", "-1")
+	out.Header().Set("Pragma", "no-cache")
 	out.Header().Set("Content-Type", "text/html")
-	fmt.Fprintf(out, "<HTML>\n<HEAD><TITLE>WRP %s</TITLE></HEAD>\n<BODY BGCOLOR=\"#F0F0F0\">", u)
-	fmt.Fprintf(out, "<FORM ACTION=\"/\">URL/Search: <INPUT TYPE=\"TEXT\" NAME=\"url\" VALUE=\"%s\" SIZE=\"40\">", u)
-	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" VALUE=\"Go\"><P>\n")
-	fmt.Fprintf(out, "ISMAP:<INPUT TYPE=\"CHECKBOX\" NAME=\"i\" %s> \n", istr)
-	fmt.Fprintf(out, "Width:<INPUT TYPE=\"TEXT\" NAME=\"w\" VALUE=\"%d\" SIZE=\"4\"> \n", w)
-	fmt.Fprintf(out, "Height:<INPUT TYPE=\"TEXT\" NAME=\"h\" VALUE=\"%d\" SIZE=\"4\"> \n", h)
-	fmt.Fprintf(out, "Scale:<INPUT TYPE=\"TEXT\" NAME=\"s\" VALUE=\"%1.2f\" SIZE=\"3\"> \n", s)
-	fmt.Fprintf(out, "Page:<INPUT TYPE=\"HIDDEN\" NAME=\"p\" VALUE=\"%d\"> \n", p)
-	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" NAME=\"pg\" VALUE=\"<<\"> %d \n", p)
-	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" NAME=\"pg\" VALUE=\">>\"> \n")
-	fmt.Fprintf(out, "</FORM><P>\n")
-	if len(u) > 4 {
-		if strings.HasPrefix(u, "http") {
-			capture(u, w, h, s, p, i, out)
-		} else {
-			capture(fmt.Sprintf("http://www.google.com/search?q=%s", url.QueryEscape(u)), w, h, s, p, i, out)
-		}
+	fmt.Fprintf(out, "<!-- Web Rendering Proxy Version %s -->\n", version)
+	fmt.Fprintf(out, "<HTML>\n<HEAD><TITLE>WRP %s</TITLE></HEAD>\n<BODY BGCOLOR=\"#F0F0F0\">\n", w.U)
+	fmt.Fprintf(out, "<FORM ACTION=\"/\" METHOD=\"POST\">\n")
+	fmt.Fprintf(out, "<INPUT TYPE=\"TEXT\" NAME=\"url\" VALUE=\"%s\" SIZE=\"10\">", w.U)
+	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" VALUE=\"Go\">\n")
+	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" NAME=\"Fn\" VALUE=\"Bk\">\n")
+	fmt.Fprintf(out, "W <INPUT TYPE=\"TEXT\" NAME=\"w\" VALUE=\"%d\" SIZE=\"4\"> \n", w.W)
+	fmt.Fprintf(out, "H <INPUT TYPE=\"TEXT\" NAME=\"h\" VALUE=\"%d\" SIZE=\"4\"> \n", w.H)
+	fmt.Fprintf(out, "S <INPUT TYPE=\"TEXT\" NAME=\"s\" VALUE=\"%1.2f\" SIZE=\"3\"> \n", w.S)
+	fmt.Fprintf(out, "C <INPUT TYPE=\"TEXT\" NAME=\"c\" VALUE=\"%d\" SIZE=\"3\">\n", w.C)
+	fmt.Fprintf(out, "K <INPUT TYPE=\"TEXT\" NAME=\"k\" VALUE=\"\" SIZE=\"4\"> \n")
+	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" NAME=\"Fn\" VALUE=\"Bs\">\n")
+	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" NAME=\"Fn\" VALUE=\"Rt\">\n")
+	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" NAME=\"Fn\" VALUE=\"&lt;\">\n")
+	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" NAME=\"Fn\" VALUE=\"^\">\n")
+	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" NAME=\"Fn\" VALUE=\"v\">\n")
+	fmt.Fprintf(out, "<INPUT TYPE=\"SUBMIT\" NAME=\"Fn\" VALUE=\"&gt;\" SIZE=\"1\">\n")
+	fmt.Fprintf(out, "</FORM><BR>\n")
+}
+
+func (w wrpReq) printFooter(out http.ResponseWriter) {
+	fmt.Fprintf(out, "\n<P><A HREF=\"/?url=https://github.com/tenox7/wrp/&w=%d&h=%d&s=%1.2f&c=%d\">"+
+		"Web Rendering Proxy Version %s</A> | <A HREF=\"/shutdown/\">Shutdown WRP</A></BODY>\n</HTML>\n", w.W, w.H, w.S, w.C, version)
+}
+
+func pageServer(out http.ResponseWriter, req *http.Request) {
+	log.Printf("%s Page Request for %s [%+v]\n", req.RemoteAddr, req.URL.Path, req.URL.RawQuery)
+	var w wrpReq
+	w.parseForm(req)
+
+	if len(w.U) > 4 {
+		w.capture(req.RemoteAddr, out)
 	} else {
-		fmt.Fprintf(out, "No URL or search query specified")
+		w.printPage(out)
+		w.printFooter(out)
+	}
+}
+
+func mapServer(out http.ResponseWriter, req *http.Request) {
+	log.Printf("%s ISMAP Request for %s [%+v]\n", req.RemoteAddr, req.URL.Path, req.URL.RawQuery)
+	w, ok := ismap[req.URL.Path]
+	if !ok {
+		fmt.Fprintf(out, "Unable to find map %s\n", req.URL.Path)
+		log.Printf("Unable to find map %s\n", req.URL.Path)
+		return
+	}
+	defer delete(ismap, req.URL.Path)
+	n, err := fmt.Sscanf(req.URL.RawQuery, "%d,%d", &w.X, &w.Y)
+	if err != nil || n != 2 {
+		fmt.Fprintf(out, "n=%d, err=%s\n", n, err)
+		log.Printf("%s ISMAP n=%d, err=%s\n", req.RemoteAddr, n, err)
+		return
+	}
+	log.Printf("%s WrpReq from ISMAP: %+v\n", req.RemoteAddr, w)
+	if len(w.U) > 4 {
+		w.capture(req.RemoteAddr, out)
+	} else {
+		w.printPage(out)
+		w.printFooter(out)
 	}
-	fmt.Fprintf(out, "</BODY>\n</HTML>\n")
 }
 
 func imgServer(out http.ResponseWriter, req *http.Request) {
 	log.Printf("%s IMG Request for %s\n", req.RemoteAddr, req.URL.Path)
-	gifbuf := gifmap[req.URL.Path]
+	gifbuf, ok := gifmap[req.URL.Path]
+	if !ok || gifbuf.Bytes() == nil {
+		fmt.Fprintf(out, "Unable to find image %s\n", req.URL.Path)
+		log.Printf("Unable to find image %s\n", req.URL.Path)
+		return
+	}
 	defer delete(gifmap, req.URL.Path)
 	out.Header().Set("Content-Type", "image/gif")
 	out.Header().Set("Content-Length", strconv.Itoa(len(gifbuf.Bytes())))
+	out.Header().Set("Cache-Control", "max-age=0")
+	out.Header().Set("Expires", "-1")
+	out.Header().Set("Pragma", "no-cache")
 	out.Write(gifbuf.Bytes())
 	out.(http.Flusher).Flush()
 }
 
-func mapServer(out http.ResponseWriter, req *http.Request) {
-	log.Printf("%s MAP Request for %s [%v]\n", req.RemoteAddr, req.URL.Path, req.URL.Query())
-}
-
-
-func haltServer(out http.ResponseWriter, req *http.Request) {
-	log.Printf("%s Shutdown request received [%s]\n", req.RemoteAddr, req.URL.Path)
-	out.Header().Set("Content-Type", "text/plain")
-	fmt.Fprintf(out, "WRP Shutdown")
-	out.(http.Flusher).Flush()
-	cancel()
-	os.Exit(0)
-}
-
-func capture(gourl string, w int64, h int64, s float64, p int64, ismap bool, out http.ResponseWriter) {
-	var nodes []*cdp.Node
-	ctxx := chromedp.FromContext(ctx)
+func (w wrpReq) capture(c string, out http.ResponseWriter) {
 	var pngbuf []byte
 	var gifbuf bytes.Buffer
-	var loc string
-	var res *runtime.RemoteObject
+	var err error
 
-	log.Printf("Processing Caputure Request for %s\n", gourl)
-
-	// Run ChromeDP Magic
-	err := chromedp.Run(ctx,
-		emulation.SetDeviceMetricsOverride(w, h, s, false),
-		chromedp.Navigate(gourl),
-		chromedp.Evaluate(fmt.Sprintf("window.scrollTo(0, %d);", p*int64(float64(h)*float64(0.9))), &res),
-		chromedp.Sleep(time.Second*1),
-		chromedp.CaptureScreenshot(&pngbuf),
-		chromedp.Location(&loc),
-		chromedp.Nodes("a", &nodes, chromedp.ByQueryAll))
+	if w.X > 0 && w.Y > 0 {
+		log.Printf("%s Mouse Click %d,%d\n", c, w.X, w.Y)
+		err = chromedp.Run(ctx, chromedp.MouseClickXY(int64(float64(w.X)/w.S), int64(float64(w.Y)/w.S)))
+	} else if len(w.F) > 0 {
+		log.Printf("%s Button %v\n", c, w.F)
+		switch w.F {
+		case "Bk":
+			err = chromedp.Run(ctx, chromedp.NavigateBack())
+		case "Bs":
+			err = chromedp.Run(ctx, chromedp.KeyEvent("\b"))
+		case "Rt":
+			err = chromedp.Run(ctx, chromedp.KeyEvent("\r"))
+		case "<":
+			err = chromedp.Run(ctx, chromedp.KeyEvent("\u0302"))
+		case "^":
+			err = chromedp.Run(ctx, chromedp.KeyEvent("\u0304"))
+		case "v":
+			err = chromedp.Run(ctx, chromedp.KeyEvent("\u0301"))
+		case ">":
+			err = chromedp.Run(ctx, chromedp.KeyEvent("\u0303"))
+		}
+	} else if len(w.K) > 0 {
+		log.Printf("%s Sending Keys: %#v\n", c, w.K)
+		err = chromedp.Run(ctx, chromedp.KeyEvent(w.K))
+	} else {
+		log.Printf("%s Processing Capture Request for %s\n", c, w.U)
+		err = chromedp.Run(ctx,
+			emulation.SetDeviceMetricsOverride(int64(float64(w.W)/w.S), int64(float64(w.H)/w.S), w.S, false),
+			chromedp.Navigate(w.U),
+		)
+	}
 
 	if err != nil {
-		log.Printf("%s", err)
-		fmt.Fprintf(out, "<BR>%s<BR>", err)
+		if err.Error() == "context canceled" {
+			log.Printf("%s Contex cancelled, try again", c)
+			fmt.Fprintf(out, "<BR>%s<BR> -- restarting, try again", err)
+			ctx, cancel = chromedp.NewContext(context.Background())
+		} else {
+			log.Printf("%s %s", c, err)
+			fmt.Fprintf(out, "<BR>%s<BR>", err)
+		}
 		return
 	}
 
-	log.Printf("Landed on: %s, Nodes: %d\n", loc, len(nodes))
+	chromedp.Run(
+		ctx, chromedp.Sleep(time.Second*2),
+		chromedp.Location(&w.U),
+	)
+	log.Printf("%s Landed on: %s\n", c, w.U)
+	w.printPage(out)
 
 	// Process Screenshot Image
+	err = chromedp.Run(ctx, chromedp.CaptureScreenshot(&pngbuf))
+	if err != nil {
+		log.Printf("%s Failed to capture screenshot: %s\n", c, err)
+		fmt.Fprintf(out, "<BR>Unable to capture screenshot:<BR>%s<BR>\n", err)
+		return
+	}
 	bytes.NewReader(pngbuf).Seek(0, 0)
 	img, err := png.Decode(bytes.NewReader(pngbuf))
 	if err != nil {
-		log.Printf("Failed to decode screenshot: %s\n", err)
+		log.Printf("%s Failed to decode screenshot: %s\n", c, err)
 		fmt.Fprintf(out, "<BR>Unable to decode page screenshot:<BR>%s<BR>\n", err)
 		return
 	}
 	gifbuf.Reset()
-	err = gif.Encode(&gifbuf, img, nil)
+	err = gif.Encode(&gifbuf, img, &gif.Options{NumColors: int(w.C), Quantizer: quantize.MedianCutQuantizer{}})
 	if err != nil {
-		log.Printf("Failed to encode GIF: %s\n", err)
+		log.Printf("%s Failed to encode GIF: %s\n", c, err)
 		fmt.Fprintf(out, "<BR>Unable to encode GIF:<BR>%s<BR>\n", err)
 		return
 	}
-	imgpath := fmt.Sprintf("/img/%04d.gif", rand.Intn(9999))
-	log.Printf("Encoded GIF image: %s, Size: %dKB\n", imgpath, len(gifbuf.Bytes())/1024)
+
+	// Compose map and gif
+	seq := rand.Intn(9999)
+	imgpath := fmt.Sprintf("/img/%04d.gif", seq)
+	mappath := fmt.Sprintf("/map/%04d.map", seq)
 	gifmap[imgpath] = gifbuf
+	ismap[mappath] = w
+	log.Printf("%s Encoded GIF image: %s, Size: %dKB, Colors: %d\n", c, imgpath, len(gifbuf.Bytes())/1024, w.C)
+	fmt.Fprintf(out, "<A HREF=\"%s\"><IMG SRC=\"%s\" BORDER=\"0\" ISMAP></A>", mappath, imgpath)
+	w.printFooter(out)
+	log.Printf("%s Done with caputure for %s\n", c, w.U)
+}
 
-	// Process Nodes
-	base, _ := url.Parse(loc)
-	if ismap {
-		fmt.Fprintf(out, "<A HREF=\"/map/123.map\"><IMG SRC=\"%s\" ALT=\"wrp\" ISMAP></A>", imgpath)
-	} else {
-		fmt.Fprintf(out, "<IMG SRC=\"%s\" ALT=\"wrp\" USEMAP=\"#map\">\n<MAP NAME=\"map\">\n", imgpath)
-	}
-
-	for _, n := range nodes {
-		b, err := dom.GetBoxModel().WithNodeID(n.NodeID).Do(cdp.WithExecutor(ctx, ctxx.Target))
-		if err != nil {
-			continue
-		}
-		tgt, err := base.Parse(n.AttributeValue("href"))
-		if err != nil {
-			continue
-		}
-		target := fmt.Sprintf("/?url=%s&w=%d&h=%d&s=%1.2f&", tgt, w, h, s) // no page# here
-
-		if len(b.Content) > 6 && len(target) > 7 {
-			if ismap {
-
-			} else {
-			fmt.Fprintf(out, "<AREA SHAPE=\"RECT\" COORDS=\"%.f,%.f,%.f,%.f\" ALT=\"%s\" TITLE=\"%s\" HREF=\"%s\">\n",
-				b.Content[0]*s, b.Content[1]*s, b.Content[4]*s, b.Content[5]*s, n.AttributeValue("href"), n.AttributeValue("href"), target)
-			}
-		}
-	}
-
-	if !ismap {
-		fmt.Fprintf(out, "</MAP>\n")
-	}
+func haltServer(out http.ResponseWriter, req *http.Request) {
+	log.Printf("%s Shutdown Request for %s\n", req.RemoteAddr, req.URL.Path)
+	out.Header().Set("Cache-Control", "max-age=0")
+	out.Header().Set("Expires", "-1")
+	out.Header().Set("Pragma", "no-cache")
+	out.Header().Set("Content-Type", "text/text")
+	fmt.Fprintf(out, "Shutting down WRP...\n")
 	out.(http.Flusher).Flush()
-	log.Printf("Done with caputure for %s\n", gourl)
+	time.Sleep(time.Second * 2)
+	cancel()
+	srv.Shutdown(context.Background())
+	os.Exit(1)
 }
 
 func main() {
-	ctx, cancel = chromedp.NewContext(context.Background())
-	defer cancel()
 	var addr string
+	var head, headless bool
+	var debug bool
 	flag.StringVar(&addr, "l", ":8080", "Listen address:port, default :8080")
+	flag.BoolVar(&head, "h", false, "Headed mode - display browser window")
+	flag.BoolVar(&debug, "d", false, "Debug ChromeDP")
 	flag.Parse()
+	if head {
+		headless = false
+	} else {
+		headless = true
+	}
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.Flag("headless", headless),
+		chromedp.Flag("hide-scrollbars", false),
+	)
+	actx, acancel := chromedp.NewExecAllocator(context.Background(), opts...)
+	defer acancel()
+	if debug {
+		ctx, cancel = chromedp.NewContext(actx, chromedp.WithDebugf(log.Printf))
+	} else {
+		ctx, cancel = chromedp.NewContext(actx)
+	}
+	defer cancel()
 	rand.Seed(time.Now().UnixNano())
+	c := make(chan os.Signal)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-c
+		log.Printf("Interrupt - shutting down.")
+		cancel()
+		srv.Shutdown(context.Background())
+		os.Exit(1)
+	}()
 	http.HandleFunc("/", pageServer)
-	http.HandleFunc("/img/", imgServer)
 	http.HandleFunc("/map/", mapServer)
+	http.HandleFunc("/img/", imgServer)
+	http.HandleFunc("/shutdown/", haltServer)
 	http.HandleFunc("/favicon.ico", http.NotFound)
-	http.HandleFunc("/halt", haltServer)
+	log.Printf("Web Rendering Proxy Version %s\n", version)
 	log.Printf("Starting WRP http server on %s\n", addr)
-	http.ListenAndServe(addr, nil)
+	srv.Addr = addr
+	err := srv.ListenAndServe()
+	if err != nil {
+		log.Fatal(err)
+	}
 }