25 Commits
4.9.0 ... 4.9.7

Author SHA1 Message Date
Antoni Sawicki
007f7f87d6 add dockerfiler for goreleaser 2026-03-18 01:18:13 -07:00
Antoni Sawicki
fb712136b5 add goreleaser 2026-03-18 01:16:39 -07:00
Antoni Sawicki
89426c969d update mod 2026-03-18 01:16:30 -07:00
Antoni Sawicki
e9e97e8146 update readme 2026-03-16 00:54:54 -07:00
Antoni Sawicki
ef43ef5ce6 fix log line 2026-03-14 21:28:40 -07:00
Antoni Sawicki
61af9046ae fixup connect mode 2026-03-14 20:59:27 -07:00
Antoni Sawicki
90dc64a625 update faq 2026-03-14 01:59:07 -07:00
Antoni Sawicki
534c26698e fix readme 2026-03-14 01:56:01 -07:00
Antoni Sawicki
2850d7f3f2 add experimental proxy mode 2026-03-14 01:27:23 -07:00
Antoni Sawicki
7a63e0444f fix url in proxy mode 2026-03-14 01:12:08 -07:00
Antoni Sawicki
6085aaf824 add map/img cache 2026-03-14 00:58:40 -07:00
Antoni Sawicki
a70f1a9ebf add experimental proxy mode 2026-03-14 00:38:47 -07:00
Antoni Sawicki
64c2a16085 replace markdown pipeline with goquery DOM walker for html mode
- Strip/simplify HTML directly instead of HTML→markdown→HTML round-trip
- Preserves tables, forms, lists, and HTML4 attributes (bgcolor, width, etc)
- Resolve relative URLs for links and images
- Add proper user-agent header for image downloads
- Parallel image downloads with sync.WaitGroup
- SVG rendering support via oksvg/rasterx
- Set correct width/height attributes on resized images
- Print media emulation to strip print-hidden elements
- Remove h2m and goldmark dependencies

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-13 00:34:50 -07:00
Antoni Sawicki
916b5ee1b7 in html mode also use chromedp to fetch content and convert outerhtml to markdown 2026-03-12 23:38:33 -07:00
Antoni Sawicki
4db89e6a3a add --profile flag 2026-03-10 23:10:59 -07:00
Antoni Sawicki
fc15e4ea44 better page load detection 2026-03-10 02:07:14 -07:00
Antoni Sawicki
619bd74891 dependency update 2026-03-10 00:19:06 -07:00
Antoni Sawicki
fd9e6f3762 add webone link 2026-03-08 00:23:37 -08:00
Antoni Sawicki
9c5c495811 bump ver 2025-08-13 00:25:43 -07:00
Antoni Sawicki
2457473706 update dependencies 2025-08-13 00:23:28 -07:00
Antoni Sawicki
79f56a920c replace fast lut with median cut quantizer for gif, only use it in gip 2025-08-13 00:21:48 -07:00
Antoni Sawicki
2042d1bb9d bump ver 2025-08-07 01:59:25 -07:00
Antoni Sawicki
ff293f2b7a bump ver to 4.9.1 2025-08-04 20:53:07 -07:00
Antoni Sawicki
e808b494ab update readme, bump ver 2025-08-04 20:52:33 -07:00
Antoni Sawicki
88fbe63fb7 remove background color detection 2025-08-04 13:31:17 -07:00
9 changed files with 666 additions and 279 deletions

92
.goreleaser.yml Normal file
View File

@@ -0,0 +1,92 @@
version: 2
project_name: wrp
before:
hooks:
- go mod tidy
builds:
- main: .
binary: wrp
env:
- CGO_ENABLED=0
goos:
- linux
- darwin
- freebsd
- openbsd
- windows
goarch:
- amd64
- arm64
- arm
goarm:
- "7"
ldflags:
- -s -w
archives:
- formats:
- tar.gz
format_overrides:
- goos: windows
formats:
- zip
dockers:
- image_templates:
- "tenox7/wrp:{{ .Tag }}-amd64"
- "tenox7/wrp:latest-amd64"
use: buildx
goarch: amd64
dockerfile: Dockerfile.goreleaser
build_flag_templates:
- "--platform=linux/amd64"
- image_templates:
- "tenox7/wrp:{{ .Tag }}-arm64"
- "tenox7/wrp:latest-arm64"
use: buildx
goarch: arm64
dockerfile: Dockerfile.goreleaser
build_flag_templates:
- "--platform=linux/arm64"
docker_manifests:
- name_template: "tenox7/wrp:{{ .Tag }}"
image_templates:
- "tenox7/wrp:{{ .Tag }}-amd64"
- "tenox7/wrp:{{ .Tag }}-arm64"
- name_template: "tenox7/wrp:latest"
image_templates:
- "tenox7/wrp:latest-amd64"
- "tenox7/wrp:latest-arm64"
nfpms:
- package_name: wrp
vendor: Antoni Sawicki
homepage: https://github.com/tenox7/wrp
maintainer: Antoni Sawicki <as@tenoware.com>
description: Web Rendering Proxy
license: Apache-2.0
formats:
- deb
- rpm
- apk
checksum:
name_template: "checksums.txt"
changelog:
sort: asc
filters:
exclude:
- "^docs:"
- "^test:"
release:
github:
owner: tenox7
name: wrp

5
Dockerfile.goreleaser Normal file
View File

@@ -0,0 +1,5 @@
FROM chromedp/headless-shell
COPY wrp /wrp
ENTRYPOINT ["/wrp"]
ENV PATH="/headless-shell:${PATH}"
LABEL maintainer="as@tenoware.com"

View File

@@ -1,9 +1,18 @@
# WRP - Web Rendering Proxy
A browser-in-browser "proxy" server that allows to use historical / vintage web browsers on the modern web. It has two modes:
A browser-in-browser proxy server that allows to use historical / vintage web browsers on the modern web.
- ISMAP "graphical" mode, renders web page in to a GIF, PNG or JPG image with clickable imagemap.
- Simple HTML "text" mode converts web page in to Markdown, then renders it into simplified HTML for old browsers.
## Modes of operation
### Access
- Browser in browser, navigate to the address of WRP and use built-in web form to naviate.
- HTTP proxy mode which offers more seamless, but also limited use.
### Rende modes
- ISMAP "graphical" mode, renders web page in to a GIF/PNG/JPG image with clickable imagemap.
- Simple HTML mode converts pages into simplified HTML for old browsers.
![Internet Explorer 1.5 doing Gmail](wrp.png)
@@ -11,18 +20,25 @@ A browser-in-browser "proxy" server that allows to use historical / vintage web
* [Download a WRP binary](https://github.com/tenox7/wrp/releases/) run it on a machine that will become your WRP gateway/server. This should be modern hardware and OS. Google Chrome / Chromium Browser is required to be preinstalled. Do not try to run WRP on an old machine like Windows XP or 98.
* Make sure you have disabled firewall or open port WRP is listening on (by default 8080).
* Point your legacy browser to `http://address:port` of the WRP server. Do not set or use it as a "proxy server".
* Point your legacy browser to `http://address:port` of the WRP server or set your proxy address there.
* Type a search string or a full http/https URL and click **Go**.
* Select whether you want to use graphical (ISMAP) or simple HTML mode.
### Proxy mode
- Try to use http:// addresses only, the proxy will automatically rewrite them to https as needed.
- https:// addresses are limited in functionality (no /suffix or path).
- You can use `/proxy.pac` automatic configuration mode.
- Proxy mode doesn't have a customizable web interface with html forms. All configuration is done via flags.
### Image Map Mode
* Adjust your screen **W**idth/**H**eight/**S**cale/**C**olors to fit in your old browser.
* Scroll web page by clicking on the in-image scroll bar on the right.
* WRP also allows **a single tall image without the vertical scrollbar** and use client scrolling. To enable this, simply height **H** to `0` . However this should not be used with old and low spec clients. Such tall images will be very large, take a lot of memory and long time to process, especially for GIFs.
* WRP also allows **a single tall image without the vertical scrollbar** and use client scrolling. To enable this, simply height **H** to `0` (or flag `-g 1152x0x216`. However this should not be used with old and low spec clients. Such tall images will be very large, take a lot of memory and long time to process, especially for GIFs.
* Do not use client browser history-back, instead use **Bk** button in the app.
* You can re-capture page screenshot without reloading by using **St** (Stop). This is useful if page didn't render fully before screenshot is taken.
* You can also reload and re-capture current page with **Re** (Reload).
* You can re-capture screenshot without reloading page by using **St** (Stop). This is useful if page didn't render fully before screenshot is taken.
* You can also reload page and re-capture screenshot with **Re** (Reload).
* To send keystrokes, fill **K** input box and press **Go**. There also are buttons for backspace, enter and arrow keys.
* The default image type GIP is a ultra fast, optimized, parallel encoded GIF type.
* If your browser supports it, prefer PNG over GIF/JPG. PNG is much faster, whereas GIF/JPG requires a lot of additional processing on both client and server to encode/decode.
@@ -161,6 +177,10 @@ $ chmod +x wrp-amd64-macos
$ ./wrp-amd64-macos
```
### Pages are chopped off
Click `st` to re-capture screenshot. You may want to increase the page delay using `-s` flag.
### Websites are blocking headless browsers
This is a well known issue. WRP has some provisions to work around it, but it's a cat and mouse game. By default WRP tries to obtain some current valid User Agent
@@ -170,21 +190,13 @@ from https://github.com/jnrbsn/user-agents rather than using the internal "Headl
$ wrp -ua="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"
```
### Why is WRP called "proxy" when it's not
WRP originally started as true http proxy. However this stopped working because the whole internet is now encrypted thanks to [Let's Encrypt](https://en.wikipedia.org/wiki/Let%27s_Encrypt). Legacy browsers do not support modern SSL/TLS certs as well as [HTTP CONNECT](https://en.wikipedia.org/wiki/HTTP_tunnel#HTTP_CONNECT_method) so this mode had to be disabled.
### Will you support http proxy mode in future?
Some efforts (ssl strip) are under way but it's very [difficult](https://en.wikipedia.org/wiki/HTTP_tunnel#HTTP_CONNECT_method) to do it correctly and the priority is rather low.
### Why isn't there a Docker image for armv6
Because https://hub.docker.com/r/chromedp/headless-shell/ doesn't have one. WRP uses that image. If you have a fork that builds for armv6 let me know.
### WTF is GIP image format
It's just GIF but optimized. Avoids dithering, uses fast color palette and parallel encoding. https://github.com/tenox7/gip
It's just GIF but optimized. Avoids dithering, uses fast color palette and parallel encoding. https://github.com/tenox7/gip
## History
@@ -200,6 +212,7 @@ It's just GIF but optimized. Avoids dithering, uses fast color palette and para
* Version 4.7 add simple html aka reader aka text mode.
* Version 4.8 add image support to simple html mode.
* Version 4.9 adds support for ultra fast, parallel encoded gif image (GIP)
* Version 4.10 re-adds support for http proxy mode
## Credits
@@ -215,6 +228,7 @@ It's just GIF but optimized. Avoids dithering, uses fast color palette and para
You may also be interested in:
* [VncFox](https://github.com/tenox7/vncfox)
* [webone](https://github.com/atauenis/webone)
* [Browservice](https://github.com/ttalvitie/browservice)
* [Browsh](https://github.com/browsh-org/browsh)
@@ -222,5 +236,5 @@ You may also be interested in:
```text
License: Apache 2.0
Copyright (c) 2013-2025 Antoni Sawicki
Copyright (c) 2013-2026 Antoni Sawicki
```

28
go.mod
View File

@@ -1,33 +1,31 @@
module github.com/tenox7/wrp
go 1.24
toolchain go1.24.0
go 1.26
require (
github.com/JohannesKaufmann/html-to-markdown v1.6.0
github.com/MaxHalford/halfgone v0.0.0-20171017091812-482157b86ccb
github.com/breml/rootcerts v0.3.1
github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327
github.com/chromedp/chromedp v0.14.0
github.com/PuerkitoBio/goquery v1.11.0
github.com/breml/rootcerts v0.3.4
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
github.com/chromedp/chromedp v0.14.2
github.com/ericpauley/go-quantize v0.0.0-20200331213906-ae555eb2afa4
github.com/lithammer/shortuuid/v4 v4.2.0
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/soniakeys/quant v1.0.0
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef
github.com/tenox7/gip v1.0.1
github.com/yuin/goldmark v1.7.13
golang.org/x/image v0.29.0
golang.org/x/image v0.36.0
golang.org/x/net v0.51.0
)
require (
github.com/PuerkitoBio/goquery v1.10.3 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/chromedp/sysutil v1.1.0 // indirect
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
github.com/google/uuid v1.6.0 // indirect
golang.org/x/net v0.42.0 // indirect
golang.org/x/sys v0.34.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/text v0.34.0 // indirect
)

80
go.sum
View File

@@ -1,25 +1,21 @@
github.com/JohannesKaufmann/html-to-markdown v1.6.0 h1:04VXMiE50YYfCfLboJCLcgqF5x+rHJnb1ssNmqpLH/k=
github.com/JohannesKaufmann/html-to-markdown v1.6.0/go.mod h1:NUI78lGg/a7vpEJTz/0uOcYMaibytE4BUOQS8k78yPQ=
github.com/MaxHalford/halfgone v0.0.0-20171017091812-482157b86ccb h1:YQ+d0g0P0F/06oDoeEgDHeZCIrnKgLxXcqYOpe8sTuU=
github.com/MaxHalford/halfgone v0.0.0-20171017091812-482157b86ccb/go.mod h1:J86XzS1wgzJPjpQmpriJ+SetP17JSQUd9l+HWQK86jA=
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/breml/rootcerts v0.3.1 h1:PTO35OcW58K2ZYtdBykCsZh9k/eRd57bY65EHrKK/xA=
github.com/breml/rootcerts v0.3.1/go.mod h1:S/PKh+4d1HUn4HQovEB8hPJZO6pUZYrIhmXBhsegfXw=
github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 h1:UQ4AU+BGti3Sy/aLU8KVseYKNALcX9UXY6DfpwQ6J8E=
github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
github.com/chromedp/chromedp v0.14.0 h1:/xE5m6wEBwivhalHwlCOyYfBcAJNwg4nLw96QiCfYr0=
github.com/chromedp/chromedp v0.14.0/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
github.com/breml/rootcerts v0.3.4 h1:9i7WNl/ctd9OEAOaTfLy//Wrlfxq/tRQ7v4okYFN9Ys=
github.com/breml/rootcerts v0.3.4/go.mod h1:S/PKh+4d1HUn4HQovEB8hPJZO6pUZYrIhmXBhsegfXw=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
github.com/ericpauley/go-quantize v0.0.0-20200331213906-ae555eb2afa4 h1:BBade+JlV/f7JstZ4pitd4tHhpN+w+6I+LyOS7B4fyU=
github.com/ericpauley/go-quantize v0.0.0-20200331213906-ae555eb2afa4/go.mod h1:H7chHJglrhPPzetLdzBleF8d22WYOv7UM/lEKYiwlKM=
github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 h1:vymEbVwYFP/L05h5TKQxvkXoKxNvTpjxYKdF1Nlwuao=
github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
@@ -29,11 +25,6 @@ github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakr
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/lithammer/shortuuid/v4 v4.2.0 h1:LMFOzVB3996a7b8aBuEXxqOBflbfPQAiVzkIcHO0h8c=
@@ -42,34 +33,21 @@ github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/soniakeys/quant v1.0.0 h1:N1um9ktjbkZVcywBVAAYpZYSHxEfJGzshHCxx/DaI0Y=
github.com/soniakeys/quant v1.0.0/go.mod h1:HI1k023QuVbD4H8i9YdfZP2munIHU4QpjsImz6Y6zds=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE=
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q=
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ=
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE=
github.com/tenox7/gip v1.0.1 h1:yRcHROzwBjV2BhCjnh1y19wIg5Ei5CTMaZ+lx9nMl3Q=
github.com/tenox7/gip v1.0.1/go.mod h1:MR/eaUKjLGkYIguDcAUrWyxG58ipjjCrzM92jwGqDno=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA=
github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas=
golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA=
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -79,15 +57,13 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -102,24 +78,20 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -131,6 +103,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
@@ -138,9 +112,3 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

181
ismap.go
View File

@@ -15,9 +15,9 @@ import (
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/chromedp/cdproto/css"
"github.com/chromedp/cdproto/emulation"
"github.com/chromedp/cdproto/input"
"github.com/chromedp/cdproto/page"
@@ -26,6 +26,59 @@ import (
"github.com/tenox7/gip"
)
type cachedImg struct {
buf bytes.Buffer
}
type cachedMap struct {
req wrpReq
}
type wrpCache struct {
sync.Mutex
imgs map[string]cachedImg
maps map[string]cachedMap
}
func (c *wrpCache) addImg(path string, buf bytes.Buffer) {
c.Lock()
defer c.Unlock()
c.imgs[path] = cachedImg{buf: buf}
}
func (c *wrpCache) getImg(path string) (bytes.Buffer, bool) {
c.Lock()
defer c.Unlock()
e, ok := c.imgs[path]
if !ok {
return bytes.Buffer{}, false
}
return e.buf, true
}
func (c *wrpCache) addMap(path string, req wrpReq) {
c.Lock()
defer c.Unlock()
c.maps[path] = cachedMap{req: req}
}
func (c *wrpCache) getMap(path string) (wrpReq, bool) {
c.Lock()
defer c.Unlock()
e, ok := c.maps[path]
if !ok {
return wrpReq{}, false
}
return e.req, true
}
func (c *wrpCache) clear() {
c.Lock()
defer c.Unlock()
c.imgs = make(map[string]cachedImg)
c.maps = make(map[string]cachedMap)
}
func chromedpStart() (context.CancelFunc, context.CancelFunc) {
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.Flag("headless", *headless),
@@ -35,14 +88,18 @@ func chromedpStart() (context.CancelFunc, context.CancelFunc) {
)
if *userAgent == "jnrbsn" {
if ua := fetchJnrbsnUserAgent(); ua != "" {
opts = append(opts, chromedp.UserAgent(ua))
*userAgent = ua
}
} else if *userAgent != "" {
}
if *userAgent != "" {
opts = append(opts, chromedp.UserAgent(*userAgent))
}
if *browserPath != "" {
opts = append(opts, chromedp.ExecPath(*browserPath))
}
if *userDataDir != "" {
opts = append(opts, chromedp.UserDataDir(*userDataDir))
}
actx, acncl = chromedp.NewExecAllocator(context.Background(), opts...)
ctx, cncl = chromedp.NewContext(actx)
return cncl, acncl
@@ -117,6 +174,46 @@ func ctxErr(err error, w io.Writer) {
fmt.Fprintln(w, "Created new context, try again")
}
func waitForRender() chromedp.ActionFunc {
return func(ctx context.Context) error {
timeout := *delay
if timeout > 5*time.Second {
timeout = 5 * time.Second
}
ch := make(chan struct{}, 1)
lctx, lcancel := context.WithCancel(ctx)
defer lcancel()
chromedp.ListenTarget(lctx, func(ev interface{}) {
if e, ok := ev.(*page.EventLifecycleEvent); ok && e.Name == "networkAlmostIdle" {
select {
case ch <- struct{}{}:
default:
}
}
})
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
select {
case <-ch:
return nil
default:
}
var ready bool
if err := chromedp.Evaluate(
`document.readyState === "complete" && Array.from(document.images).every(i => i.complete)`,
&ready,
).Do(ctx); err != nil {
return nil
}
if ready {
return nil
}
time.Sleep(100 * time.Millisecond)
}
return nil
}
}
// https://github.com/chromedp/chromedp/issues/979
func chromedpCaptureScreenshot(res *[]byte, h int64) chromedp.Action {
if res == nil {
@@ -135,15 +232,12 @@ func chromedpCaptureScreenshot(res *[]byte, h int64) chromedp.Action {
// Capture Screenshot using CDP
func (rq *wrpReq) captureScreenshot() {
var styles []*css.ComputedStyleProperty
var r, g, b int
var bgColorSet bool
wrpCach.clear()
var h int64
var pngCap []byte
chromedp.Run(ctx,
emulation.SetDeviceMetricsOverride(int64(float64(rq.width)/rq.zoom), 10, rq.zoom, false),
chromedp.Location(&rq.url),
chromedp.ComputedStyle("body", &styles, chromedp.ByQuery),
chromedp.ActionFunc(func(ctx context.Context) error {
_, _, _, _, _, s, err := page.GetLayoutMetrics().Do(ctx)
if err == nil {
@@ -152,27 +246,17 @@ func (rq *wrpReq) captureScreenshot() {
return nil
}),
)
if rq.proxy {
rq.url = strings.Replace(rq.url, "https://", "http://", 1)
}
log.Printf("%s Landed on: %s, Height: %v\n", rq.r.RemoteAddr, rq.url, h)
for _, style := range styles {
if style.Name != "background-color" {
continue
}
fmt.Sscanf(style.Value, "rgb(%d,%d,%d)", &r, &g, &b)
bgColorSet = true
break
}
if !bgColorSet {
r = 255
g = 255
b = 255
}
height := int64(float64(rq.height) / rq.zoom)
if rq.height == 0 && h > 0 {
height = h + 30
}
chromedp.Run(
ctx, emulation.SetDeviceMetricsOverride(int64(float64(rq.width)/rq.zoom), height, rq.zoom, false),
chromedp.Sleep(*delay), // TODO(tenox): find a better way to determine if page is rendered
waitForRender(),
)
// Capture screenshot...
ctxErr(chromedp.Run(ctx, chromedpCaptureScreenshot(&pngCap, rq.height)), rq.w)
@@ -185,7 +269,7 @@ func (rq *wrpReq) captureScreenshot() {
}
imgPath := fmt.Sprintf("/img/%s.%s", seq, imgExt)
mapPath := fmt.Sprintf("/map/%s.map", seq)
ismap[mapPath] = *rq
wrpCach.addMap(mapPath, *rq)
var sSize string
var iW, iH int
switch rq.imgType {
@@ -204,14 +288,14 @@ func (rq *wrpReq) captureScreenshot() {
fmt.Fprintf(rq.w, "<BR>Unable to encode GIP:<BR>%s<BR>\n", err)
return
}
img[imgPath] = gipBuf
wrpCach.addImg(imgPath, gipBuf)
sSize = fmt.Sprintf("%.0f KB", float32(len(gipBuf.Bytes()))/1024.0)
iW = i.Bounds().Max.X
iH = i.Bounds().Max.Y
log.Printf("%s Encoded GIP image: %s, Size: %s, Res: %dx%d, Time: %vms\n", rq.r.RemoteAddr, imgPath, sSize, iW, iH, time.Since(st).Milliseconds())
case "png":
pngBuf := bytes.NewBuffer(pngCap)
img[imgPath] = *pngBuf
wrpCach.addImg(imgPath, *pngBuf)
cfg, _, _ := image.DecodeConfig(pngBuf)
sSize = fmt.Sprintf("%.0f KB", float32(len(pngBuf.Bytes()))/1024.0)
iW = cfg.Width
@@ -232,7 +316,7 @@ func (rq *wrpReq) captureScreenshot() {
fmt.Fprintf(rq.w, "<BR>Unable to encode GIF:<BR>%s<BR>\n", err)
return
}
img[imgPath] = gifBuf
wrpCach.addImg(imgPath, gifBuf)
sSize = fmt.Sprintf("%.0f KB", float32(len(gifBuf.Bytes()))/1024.0)
iW = i.Bounds().Max.X
iH = i.Bounds().Max.Y
@@ -252,27 +336,34 @@ func (rq *wrpReq) captureScreenshot() {
fmt.Fprintf(rq.w, "<BR>Unable to encode JPG:<BR>%s<BR>\n", err)
return
}
img[imgPath] = jpgBuf
wrpCach.addImg(imgPath, jpgBuf)
sSize = fmt.Sprintf("%.0f KB", float32(len(jpgBuf.Bytes()))/1024.0)
iW = i.Bounds().Max.X
iH = i.Bounds().Max.Y
log.Printf("%s Encoded JPG image: %s, Size: %s, Quality: %d, Res: %dx%d, Time: %vms\n", rq.r.RemoteAddr, imgPath, sSize, *defJpgQual, iW, iH, time.Since(st).Milliseconds())
}
rq.printUI(uiParams{
bgColor: fmt.Sprintf("#%02X%02X%02X", r, g, b),
pageHeight: fmt.Sprintf("%d PX", h),
imgSize: sSize,
imgURL: imgPath,
mapURL: mapPath,
imgWidth: iW,
imgHeight: iH,
})
if rq.proxy {
rq.w.Header().Set("Content-Type", "text/html")
fmt.Fprintf(rq.w, "<HTML><HEAD>%s<TITLE>%s</TITLE></HEAD><BODY BGCOLOR=\"%s\">"+
"<A HREF=\"%s\"><IMG SRC=\"%s\" BORDER=\"0\" WIDTH=\"%d\" HEIGHT=\"%d\" ISMAP></A>"+
"</BODY></HTML>", rq.baseTag(), rq.url, *bgColor, mapPath, imgPath, iW, iH)
} else {
rq.printUI(uiParams{
bgColor: *bgColor,
pageHeight: fmt.Sprintf("%d PX", h),
imgSize: sSize,
imgURL: imgPath,
mapURL: mapPath,
imgWidth: iW,
imgHeight: iH,
})
}
log.Printf("%s Done with capture for %s\n", rq.r.RemoteAddr, rq.url)
}
func mapServer(w http.ResponseWriter, r *http.Request) {
log.Printf("%s ISMAP Request for %s [%+v]\n", r.RemoteAddr, r.URL.Path, r.URL.RawQuery)
rq, ok := ismap[r.URL.Path]
rq, ok := wrpCach.getMap(r.URL.Path)
rq.r = r
rq.w = w
if !ok {
@@ -280,9 +371,6 @@ func mapServer(w http.ResponseWriter, r *http.Request) {
log.Printf("Unable to find map %s\n", r.URL.Path)
return
}
if !*noDel {
defer delete(ismap, r.URL.Path)
}
n, err := fmt.Sscanf(r.URL.RawQuery, "%d,%d", &rq.mouseX, &rq.mouseY)
if err != nil || n != 2 {
fmt.Fprintf(w, "n=%d, err=%s\n", n, err)
@@ -291,25 +379,30 @@ func mapServer(w http.ResponseWriter, r *http.Request) {
}
log.Printf("%s WrpReq from ISMAP: %+v\n", r.RemoteAddr, rq)
if len(rq.url) < 4 {
rq.printUI(uiParams{bgColor: "#FFFFFF"})
rq.printUI(uiParams{})
return
}
rq.navigate() // TODO: if error from navigate do not capture
if rq.proxy {
chromedp.Run(ctx, waitForRender())
var loc string
chromedp.Run(ctx, chromedp.Location(&loc))
loc = strings.Replace(loc, "https://", "http://", 1)
http.Redirect(w, r, loc, http.StatusFound)
return
}
rq.captureScreenshot()
}
// TODO: merge this with html mode IMGZ
func imgServerMap(w http.ResponseWriter, r *http.Request) {
log.Printf("%s IMG Request for %s\n", r.RemoteAddr, r.URL.Path)
imgBuf, ok := img[r.URL.Path]
imgBuf, ok := wrpCach.getImg(r.URL.Path)
if !ok || imgBuf.Bytes() == nil {
fmt.Fprintf(w, "Unable to find image %s\n", r.URL.Path)
log.Printf("%s Unable to find image %s\n", r.RemoteAddr, r.URL.Path)
return
}
if !*noDel {
defer delete(img, r.URL.Path)
}
switch {
case strings.HasSuffix(r.URL.Path, ".gif"):
w.Header().Set("Content-Type", "image/gif")

351
shtml.go
View File

@@ -1,23 +1,6 @@
// WRP TXT / Simple HTML Mode Routines
// WRP Simple HTML Mode Routines
package main
// TODO:
// - add image processing times counter to the footer
// - img cache w/garbage collector / test back/button behavior in old browsers
// - add referer header
// - svg support
// - incorrect cert support in both markdown and image download
// - unify cdp and txt image handlers
// - use goroutiness to process images
// - get inner html from chromedp instead of html2markdown
//
// - BUG: DomainFromURL always prefixes with http instead of https
// reproduces on vsi vms docs
// - BUG: markdown table errors
// reproduces on hacker news
// - BUG: captcha errors using html to markdown, perhaps use cdp inner html + downloaded images
// reproduces on https://www.cnn.com/cnn-underscored/electronics
import (
"bytes"
"encoding/base64"
@@ -30,23 +13,22 @@ import (
"io"
"log"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
h2m "github.com/JohannesKaufmann/html-to-markdown"
"github.com/JohannesKaufmann/html-to-markdown/plugin"
"github.com/PuerkitoBio/goquery"
"github.com/chromedp/cdproto/emulation"
"github.com/chromedp/chromedp"
"github.com/lithammer/shortuuid/v4"
"github.com/nfnt/resize"
"github.com/srwiley/oksvg"
"github.com/srwiley/rasterx"
"github.com/tenox7/gip"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
"golang.org/x/image/webp"
"golang.org/x/net/html"
)
var imgStor imageStore
@@ -90,62 +72,114 @@ func (i *imageStore) del(id string) {
delete(i.img, id)
}
func fetchImage(id, url, imgType string, maxSize, imgOpt int) (int, error) {
log.Printf("Downloading IMGZ URL=%q for ID=%q", url, id)
func (i *imageStore) clear() {
i.Lock()
defer i.Unlock()
i.img = make(map[string]imageContainer)
}
func fetchImage(id, imgURL, imgType string, maxSize, imgOpt int) (int, int, int, error) {
log.Printf("Downloading IMGZ URL=%q for ID=%q", imgURL, id)
var in []byte
var err error
switch url[:4] {
if len(imgURL) < 4 {
return 0, 0, 0, fmt.Errorf("image URL too short: %q", imgURL)
}
switch imgURL[:4] {
case "http":
r, err := http.Get(url) // TODO: possibly set a header "referer" here
req, err := http.NewRequest("GET", imgURL, nil)
if err != nil {
return 0, fmt.Errorf("Error downloading %q: %v", url, err)
return 0, 0, 0, fmt.Errorf("Error creating request for %q: %v", imgURL, err)
}
if *userAgent != "" {
req.Header.Set("User-Agent", *userAgent)
}
r, err := http.DefaultClient.Do(req)
if err != nil {
return 0, 0, 0, fmt.Errorf("Error downloading %q: %v", imgURL, err)
}
if r.StatusCode != http.StatusOK {
return 0, fmt.Errorf("Error %q HTTP Status Code: %v", url, r.StatusCode)
return 0, 0, 0, fmt.Errorf("Error %q HTTP Status Code: %v", imgURL, r.StatusCode)
}
defer r.Body.Close()
in, err = io.ReadAll(r.Body)
if err != nil {
return 0, fmt.Errorf("Error reading %q: %v", url, err)
return 0, 0, 0, fmt.Errorf("Error reading %q: %v", imgURL, err)
}
case "data":
idx := strings.Index(url, ",")
idx := strings.Index(imgURL, ",")
if idx < 1 {
return 0, fmt.Errorf("image is embeded but unable to find coma: %q", url)
return 0, 0, 0, fmt.Errorf("image is embeded but unable to find coma: %q", imgURL)
}
in, err = base64.StdEncoding.DecodeString(url[idx+1:])
in, err = base64.StdEncoding.DecodeString(imgURL[idx+1:])
if err != nil {
return 0, fmt.Errorf("error decoding image from url embed: %q: %v", url, err)
return 0, 0, 0, fmt.Errorf("error decoding image from url embed: %q: %v", imgURL, err)
}
default:
return 0, 0, 0, fmt.Errorf("unsupported image URL scheme: %q", imgURL)
}
out, err := smallImg(in, imgType, maxSize, imgOpt)
out, w, h, err := smallImg(in, imgType, maxSize, imgOpt)
if err != nil {
return 0, fmt.Errorf("Error scaling down image: %v", err)
return 0, 0, 0, fmt.Errorf("Error scaling down %q: %v", imgURL, err)
}
imgStor.add(id, url, out)
return len(out), nil
imgStor.add(id, imgURL, out)
return len(out), w, h, nil
}
func smallImg(src []byte, imgType string, maxSize, imgOpt int) ([]byte, error) {
func decodeSVG(src []byte, maxSize int) (image.Image, error) {
icon, err := oksvg.ReadIconStream(bytes.NewReader(src))
if err != nil {
return nil, err
}
w, h := int(icon.ViewBox.W), int(icon.ViewBox.H)
if w <= 0 || h <= 0 {
w, h = maxSize, maxSize
}
if w > maxSize || h > maxSize {
ratio := float64(maxSize) / float64(max(w, h))
w = int(float64(w) * ratio)
h = int(float64(h) * ratio)
}
if w <= 0 {
w = 1
}
if h <= 0 {
h = 1
}
icon.SetTarget(0, 0, float64(w), float64(h))
rgba := image.NewRGBA(image.Rect(0, 0, w, h))
icon.Draw(rasterx.NewDasher(w, h, rasterx.NewScannerGV(w, h, rgba, rgba.Bounds())), 1)
return rgba, nil
}
func isSVG(src []byte) bool {
s := strings.TrimSpace(string(src[:min(len(src), 256)]))
return strings.HasPrefix(s, "<svg") || strings.HasPrefix(s, "<?xml")
}
func smallImg(src []byte, imgType string, maxSize, imgOpt int) ([]byte, int, int, error) {
t := http.DetectContentType(src)
var err error
var img image.Image
switch t {
case "image/png":
switch {
case t == "image/png":
img, err = png.Decode(bytes.NewReader(src))
case "image/gif":
case t == "image/gif":
img, err = gif.Decode(bytes.NewReader(src))
case "image/jpeg":
case t == "image/jpeg":
img, err = jpeg.Decode(bytes.NewReader(src))
case "image/webp":
case t == "image/webp":
img, err = webp.Decode(bytes.NewReader(src))
default: // TODO: also add svg
case t == "image/svg+xml", isSVG(src):
img, err = decodeSVG(src, maxSize)
default:
err = errors.New("unknown content type: " + t)
}
if err != nil {
return nil, fmt.Errorf("image decode problem: %v", err)
return nil, 0, 0, fmt.Errorf("image decode problem: %v", err)
}
img = resize.Thumbnail(uint(maxSize), uint(maxSize), img, resize.NearestNeighbor)
b := img.Bounds()
var outBuf bytes.Buffer
switch imgType {
case "gip":
@@ -158,81 +192,193 @@ func smallImg(src []byte, imgType string, maxSize, imgOpt int) ([]byte, error) {
err = jpeg.Encode(&outBuf, img, &jpeg.Options{Quality: imgOpt})
}
if err != nil {
return nil, fmt.Errorf("gif encode problem: %v", err)
return nil, 0, 0, fmt.Errorf("image encode problem: %v", err)
}
return outBuf.Bytes(), nil
return outBuf.Bytes(), b.Dx(), b.Dy(), nil
}
type astTransformer struct {
imgType string
maxSize int
imgOpt int
totSize int
var removeElements = []string{
"script", "style", "link", "meta", "noscript", "iframe",
"svg", "canvas", "video", "audio", "source", "picture",
"template", "slot", "dialog", "portal",
}
func (t *astTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
if link, ok := n.(*ast.Link); ok && entering {
link.Destination = append([]byte("/?m=html&t="+t.imgType+"&s="+strconv.Itoa(t.maxSize)+"&url="), link.Destination...)
}
if img, ok := n.(*ast.Image); ok && entering {
var imgExt string
if t.imgType == "gip" {
imgExt = "gif"
} else {
imgExt = t.imgType
}
seq := shortuuid.New() + "." + imgExt
size, err := fetchImage(seq, string(img.Destination), t.imgType, t.maxSize, t.imgOpt) // TODO: use goroutines with waitgroup
if err != nil {
log.Print(err)
n.Parent().RemoveChildren(n)
return ast.WalkContinue, nil
}
img.Destination = []byte(imgZpfx + seq)
t.totSize += size
}
return ast.WalkContinue, nil
})
var renameToDiv = map[string]bool{
"section": true, "article": true, "nav": true,
"header": true, "footer": true, "aside": true,
"main": true, "figure": true, "figcaption": true,
"details": true, "summary": true, "hgroup": true,
"mark": true, "time": true, "search": true,
}
func (rq *wrpReq) captureMarkdown() {
log.Printf("Processing Markdown conversion request for %v", rq.url)
// TODO: bug - DomainFromURL always prefixes with http:// instead of https
// this causes issues on some websites, fix or write a smarter DomainFromURL
c := h2m.NewConverter(h2m.DomainFromURL(rq.url), true, nil)
c.Use(plugin.GitHubFlavored())
md, err := c.ConvertURL(rq.url) // We could also get inner html from chromedp
var keepAttrs = map[string]bool{
"href": true, "src": true, "alt": true, "title": true,
"width": true, "height": true, "border": true,
"cellpadding": true, "cellspacing": true,
"bgcolor": true, "background": true,
"align": true, "valign": true,
"colspan": true, "rowspan": true, "nowrap": true,
"name": true, "value": true, "type": true,
"action": true, "method": true, "enctype": true,
"size": true, "maxlength": true,
"checked": true, "selected": true, "multiple": true,
"disabled": true, "readonly": true,
"placeholder": true, "for": true,
"rows": true, "cols": true,
"color": true, "face": true,
}
func resolveURL(raw string, base *url.URL) string {
if raw == "" {
return ""
}
ref, err := url.Parse(raw)
if err != nil {
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
return
return raw
}
return base.ResolveReference(ref).String()
}
func simplifyDOM(doc *goquery.Document, rq *wrpReq) int {
doc.Find(strings.Join(removeElements, ", ")).Remove()
doc.Find("*").Each(func(i int, s *goquery.Selection) {
for _, n := range s.Nodes {
if n.Type != html.ElementNode {
return
}
if renameToDiv[n.Data] {
n.Data = "div"
}
var keep []html.Attribute
for _, a := range n.Attr {
if keepAttrs[a.Key] {
keep = append(keep, a)
}
}
n.Attr = keep
}
})
imgExt := rq.imgType
if imgExt == "gip" {
imgExt = "gif"
}
log.Printf("Got %v bytes md from %v", len(md), rq.url)
var imgOpt int
switch rq.imgType {
case "jpg":
imgOpt = int(rq.jQual)
case "gif":
imgOpt = int(rq.nColors)
case "gip":
imgOpt = 0
}
t := &astTransformer{imgType: rq.imgType, maxSize: int(rq.maxSize), imgOpt: imgOpt}
gm := goldmark.New(
goldmark.WithExtensions(extension.GFM),
goldmark.WithParserOptions(parser.WithASTTransformers(util.Prioritized(t, 100))),
baseURL, _ := url.Parse(rq.url)
wrpParams := fmt.Sprintf("m=html&t=%s&s=%d", rq.imgType, rq.maxSize)
doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
href, exists := s.Attr("href")
if !exists || href == "" {
return
}
if strings.HasPrefix(href, "#") || strings.HasPrefix(href, "javascript:") {
return
}
abs := resolveURL(href, baseURL)
if rq.proxy {
abs = strings.Replace(abs, "https://", "http://", 1)
s.SetAttr("href", abs)
} else {
s.SetAttr("href", "/?"+wrpParams+"&url="+url.QueryEscape(abs))
}
})
type imgJob struct {
sel *goquery.Selection
seq string
abs string
}
var jobs []imgJob
doc.Find("img[src]").Each(func(i int, s *goquery.Selection) {
src, exists := s.Attr("src")
if !exists || src == "" {
s.Remove()
return
}
abs := resolveURL(src, baseURL)
seq := shortuuid.New() + "." + imgExt
jobs = append(jobs, imgJob{sel: s, seq: seq, abs: abs})
})
var wg sync.WaitGroup
var mu sync.Mutex
var totSize int
for i := range jobs {
wg.Add(1)
go func(j imgJob) {
defer wg.Done()
size, w, h, err := fetchImage(j.seq, j.abs, rq.imgType, int(rq.maxSize), imgOpt)
mu.Lock()
defer mu.Unlock()
if err != nil {
log.Print(err)
j.sel.Remove()
return
}
j.sel.SetAttr("src", imgZpfx+j.seq)
j.sel.SetAttr("width", strconv.Itoa(w))
j.sel.SetAttr("height", strconv.Itoa(h))
totSize += size
}(jobs[i])
}
wg.Wait()
return totSize
}
func (rq *wrpReq) captureMarkdown() {
imgStor.clear()
log.Printf("Processing simple HTML conversion for %v", rq.url)
var outerHTML string
err := chromedp.Run(ctx,
waitForRender(),
emulation.SetEmulatedMedia().WithMedia("print"),
chromedp.Evaluate(`(function(){document.querySelectorAll('*').forEach(function(e){if(getComputedStyle(e).display==='none')e.remove()})})()`, nil),
chromedp.OuterHTML("html", &outerHTML, chromedp.ByQuery),
emulation.SetEmulatedMedia().WithMedia(""),
)
var ht bytes.Buffer
err = gm.Convert([]byte(md), &ht)
if err != nil {
log.Printf("Failed to get OuterHTML via CDP: %v", err)
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
return
}
log.Printf("Got %v bytes HTML from CDP for %v", len(outerHTML), rq.url)
doc, err := goquery.NewDocumentFromReader(strings.NewReader(outerHTML))
if err != nil {
log.Printf("Failed to parse HTML: %v", err)
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
return
}
totSize := simplifyDOM(doc, rq)
body := doc.Find("body")
simplified, err := body.Html()
if err != nil {
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
return
}
log.Printf("Rendered %v bytes html for %v", len(ht.String()), rq.url)
log.Printf("Simplified to %v bytes html for %v", len(simplified), rq.url)
if rq.proxy {
rq.w.Header().Set("Content-Type", "text/html")
fmt.Fprintf(rq.w, "<HTML><HEAD>%s<TITLE>%s</TITLE></HEAD><BODY BGCOLOR=\"%s\">%s</BODY></HTML>",
rq.baseTag(), rq.url, *bgColor, string(asciify([]byte(simplified))))
return
}
rq.printUI(uiParams{
text: string(asciify([]byte(ht.String()))),
bgColor: "#FFFFFF",
imgSize: fmt.Sprintf("%.0f KB", float32(t.totSize)/1024.0),
text: string(asciify([]byte(simplified))),
bgColor: *bgColor,
imgSize: fmt.Sprintf("%.0f KB", float32(totSize)/1024.0),
})
}
@@ -245,7 +391,6 @@ func imgServerTxt(w http.ResponseWriter, r *http.Request) {
log.Printf("%s IMGZ error for %s: %v", r.RemoteAddr, r.URL.Path, err)
return
}
imgStor.del(id)
w.Header().Set("Content-Type", http.DetectContentType(img))
w.Header().Set("Content-Length", strconv.Itoa(len(img)))
w.Write(img)

53
util.go
View File

@@ -4,7 +4,7 @@ package main
import (
"encoding/json"
"image"
"image/color/palette"
"image/color"
"log"
"net"
"net/http"
@@ -12,21 +12,22 @@ import (
"time"
"github.com/MaxHalford/halfgone"
"github.com/soniakeys/quant/median"
"github.com/ericpauley/go-quantize/quantize"
)
func printMyIPs(b string) {
ap := strings.Split(b, ":")
if len(ap) < 1 {
if len(ap) < 2 {
log.Fatal("Wrong format of ipaddress:port")
}
log.Printf("Listen address: %v", b)
port := ap[len(ap)-1]
if ap[0] != "" && ap[0] != "0.0.0.0" {
log.Printf("Listen address: %v", b)
return
}
a, err := net.InterfaceAddrs()
if err != nil {
log.Print("Unable to get interfaces: ", err)
log.Printf("Listen address: %v", b)
return
}
var m string
@@ -35,46 +36,26 @@ func printMyIPs(b string) {
if !ok || n.IP.IsLoopback() || strings.Contains(n.IP.String(), ":") {
continue
}
m = m + n.IP.String() + " "
m += n.IP.String() + ":" + port + " "
}
log.Print("My IP addresses: ", m)
log.Printf("Listen address: %v", m)
}
func gifPalette(i image.Image, n int64) image.Image {
switch n {
case 2:
i = halfgone.FloydSteinbergDitherer{}.Apply(halfgone.ImageToGray(i))
case 216:
var FastGifLut = [256]int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5}
r := i.Bounds()
// NOTE: the color index computation below works only for palette.WebSafe!
p := image.NewPaletted(r, palette.WebSafe)
if i64, ok := i.(image.RGBA64Image); ok {
for y := r.Min.Y; y < r.Max.Y; y++ {
for x := r.Min.X; x < r.Max.X; x++ {
c := i64.RGBA64At(x, y)
r6 := FastGifLut[c.R>>8]
g6 := FastGifLut[c.G>>8]
b6 := FastGifLut[c.B>>8]
p.SetColorIndex(x, y, uint8(36*r6+6*g6+b6))
}
}
} else {
for y := r.Min.Y; y < r.Max.Y; y++ {
for x := r.Min.X; x < r.Max.X; x++ {
c := i.At(x, y)
r, g, b, _ := c.RGBA()
r6 := FastGifLut[r&0xff]
g6 := FastGifLut[g&0xff]
b6 := FastGifLut[b&0xff]
p.SetColorIndex(x, y, uint8(36*r6+6*g6+b6))
}
default:
q := quantize.MedianCutQuantizer{}
p := q.Quantize(make([]color.Color, 0, int(n)), i)
bounds := i.Bounds()
quantized := image.NewPaletted(bounds, p)
for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
for x := bounds.Min.X; x < bounds.Max.X; x++ {
quantized.Set(x, y, i.At(x, y))
}
}
i = p
default:
q := median.Quantizer(n)
i = q.Paletted(i)
i = quantized
}
return i
}

107
wrp.go
View File

@@ -7,7 +7,6 @@
package main
import (
"bytes"
"context"
"embed"
"flag"
@@ -26,32 +25,33 @@ import (
"time"
_ "github.com/breml/rootcerts"
"github.com/chromedp/chromedp"
)
const version = "4.8.2"
const version = "4.9.3"
var (
addr = flag.String("l", ":8080", "Listen address:port, default :8080")
headless = flag.Bool("h", true, "Headless mode / hide browser window (default true)")
noDel = flag.Bool("n", false, "Do not free maps and images after use")
defType = flag.String("t", "gip", "Image type: gip|png|gif|jpg")
wrpMode = flag.String("m", "ismap", "WRP Mode: ismap|html")
defImgSize = flag.Int64("is", 200, "html mode default image size")
defJpgQual = flag.Int64("q", 75, "Jpeg image quality, default 75%") // TODO: this should be form dropdown when jpeg is selected as image type
fgeom = flag.String("g", "1152x600x216", "Geometry: width x height x colors, height can be 0 for unlimited")
htmFnam = flag.String("ui", "wrp.html", "HTML template file for the UI")
delay = flag.Duration("s", 2*time.Second, "Delay/sleep after page is rendered and before screenshot is taken")
delay = flag.Duration("s", 5*time.Second, "Timeout for waiting for the page to render before screenshot")
userAgent = flag.String("ua", "jnrbsn", "override chrome user agent (jnrbsn=fetch from API, empty=default)")
browserPath = flag.String("b", "", "browser executable path (e.g., /Applications/Brave Browser.app/Contents/MacOS/Brave Browser)")
searchEng = flag.String("se", "https://duckduckgo.com/search?q=", "Search engine string")
userDataDir = flag.String("profile", "", "Chrome user data dir for persistent cookies/sessions")
bgColor = flag.String("bgcolor", "#F0F0F0", "Background color for WRP UI")
)
var (
srv http.Server
actx, ctx context.Context
acncl, cncl context.CancelFunc
img = make(map[string]bytes.Buffer)
ismap = make(map[string]wrpReq)
wrpCach wrpCache
defGeom geom
htmlTmpl *template.Template
)
@@ -117,10 +117,21 @@ type wrpReq struct {
imgType string
wrpMode string
maxSize int64
proxy bool
w http.ResponseWriter
r *http.Request
}
func (rq *wrpReq) baseTag() string {
if rq.r.Method != "CONNECT" {
return ""
}
if addr := rq.r.Context().Value(http.LocalAddrContextKey); addr != nil {
return fmt.Sprintf(`<BASE HREF="http://%v/">`, addr)
}
return ""
}
func (rq *wrpReq) parseForm() {
rq.r.ParseForm()
rq.wrpMode = rq.r.FormValue("m")
@@ -171,7 +182,7 @@ func (rq *wrpReq) printUI(p uiParams) {
rq.w.Header().Set("Pragma", "no-cache")
rq.w.Header().Set("Content-Type", "text/html")
if p.bgColor == "" {
p.bgColor = "#FFFFFF"
p.bgColor = *bgColor
}
data := uiData{
Version: version,
@@ -199,7 +210,59 @@ func (rq *wrpReq) printUI(p uiParams) {
}
}
func proxyServer(w http.ResponseWriter, r *http.Request) {
var purl string
switch {
case r.Method == "CONNECT":
purl = "https://" + r.Host
case r.URL.Scheme == "":
purl = "http://" + r.Host + r.URL.RequestURI()
default:
purl = r.URL.String()
}
log.Printf("%s Proxy Request for %s\n", r.RemoteAddr, purl)
rq := wrpReq{
r: r,
w: w,
url: purl,
width: defGeom.w,
height: defGeom.h,
nColors: defGeom.c,
zoom: 1.0,
imgType: *defType,
wrpMode: *wrpMode,
maxSize: *defImgSize,
jQual: *defJpgQual,
proxy: true,
}
var currentURL string
chromedp.Run(ctx, chromedp.Location(&currentURL))
currentURL = strings.Replace(currentURL, "https://", "http://", 1)
if currentURL != strings.Replace(rq.url, "https://", "http://", 1) {
rq.navigate()
}
rq.url = strings.Replace(rq.url, "https://", "http://", 1)
if r.Method == "CONNECT" {
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusBadGateway)
}
if rq.wrpMode == "html" {
rq.captureMarkdown()
return
}
rq.captureScreenshot()
}
func isProxyRequest(r *http.Request) bool {
return r.URL.IsAbs() || r.Method == "CONNECT"
}
func pageServer(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s %s [%+v]\n", r.RemoteAddr, r.Method, r.URL, r.Host)
if isProxyRequest(r) {
proxyServer(w, r)
return
}
log.Printf("%s Page Request for %s [%+v]\n", r.RemoteAddr, r.URL.Path, r.URL.RawQuery)
rq := wrpReq{
r: r,
@@ -207,7 +270,7 @@ func pageServer(w http.ResponseWriter, r *http.Request) {
}
rq.parseForm()
if len(rq.url) < 4 {
rq.printUI(uiParams{bgColor: "#FFFFFF"})
rq.printUI(uiParams{})
return
}
rq.navigate() // TODO: if error from navigate do not capture
@@ -218,6 +281,22 @@ func pageServer(w http.ResponseWriter, r *http.Request) {
rq.captureScreenshot()
}
func pacServer(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig")
fmt.Fprintf(w, `function FindProxyForURL(url, host) {
if (isPlainHostName(host) ||
host == "localhost" ||
isInNet(host, "127.0.0.0", "255.0.0.0") ||
isInNet(host, "10.0.0.0", "255.0.0.0") ||
isInNet(host, "172.16.0.0", "255.240.0.0") ||
isInNet(host, "192.168.0.0", "255.255.0.0") ||
isInNet(host, "169.254.0.0", "255.255.0.0"))
return "DIRECT";
return "PROXY %s";
}
`, r.Host)
}
func haltServer(w http.ResponseWriter, r *http.Request) {
log.Printf("%s Shutdown Request for %s\n", r.RemoteAddr, r.URL.Path)
w.Header().Set("Content-Type", "text/plain")
@@ -271,6 +350,7 @@ func main() {
*addr = ":" + os.Getenv(("PORT"))
}
printMyIPs(*addr)
log.Printf("Default mode: %v", *wrpMode)
n, err := fmt.Sscanf(*fgeom, "%dx%dx%d", &defGeom.w, &defGeom.h, &defGeom.c)
if err != nil || n != 3 {
log.Fatalf("Unable to parse -g geometry flag / %s", err)
@@ -291,10 +371,14 @@ func main() {
os.Exit(1)
}()
wrpCach.imgs = make(map[string]cachedImg)
wrpCach.maps = make(map[string]cachedMap)
http.HandleFunc("/", pageServer)
http.HandleFunc("/map/", mapServer)
http.HandleFunc("/img/", imgServerMap)
http.HandleFunc(imgZpfx, imgServerTxt)
http.HandleFunc("/proxy.pac", pacServer)
http.HandleFunc("/shutdown/", haltServer)
http.HandleFunc("/favicon.ico", http.NotFound)
@@ -307,6 +391,13 @@ func main() {
log.Print("Starting WRP http server")
srv.Addr = *addr
srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "CONNECT" {
pageServer(w, r)
return
}
http.DefaultServeMux.ServeHTTP(w, r)
})
err = srv.ListenAndServe()
if err != nil {
log.Fatal(err)