mirror of
https://github.com/tenox7/wrp.git
synced 2026-02-14 13:46:32 +00:00
Compare commits
35 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6e402029a | ||
|
|
9f7107c00b | ||
|
|
500ad0d19a | ||
|
|
b5747f52e7 | ||
|
|
9f21d8d06e | ||
|
|
b0f4170c6c | ||
|
|
407907bece | ||
|
|
f838caa328 | ||
|
|
a8b8a557f7 | ||
|
|
d036914fcb | ||
|
|
701240eb7c | ||
|
|
1d9af26604 | ||
|
|
5ed55c387c | ||
|
|
0e3192b69f | ||
|
|
91870f5724 | ||
|
|
c1ffd71e25 | ||
|
|
26b3c21aa7 | ||
|
|
8cec22eb90 | ||
|
|
e64f413c76 | ||
|
|
94d0e0d128 | ||
|
|
b546b5cbae | ||
|
|
a2a4152b86 | ||
|
|
db77479bad | ||
|
|
33436333cf | ||
|
|
866750700b | ||
|
|
3f445abd14 | ||
|
|
d7eb89e135 | ||
|
|
bf946f0f63 | ||
|
|
7673e15b9e | ||
|
|
d00b1d5d7f | ||
|
|
3e8f109edd | ||
|
|
9a0e5809c1 | ||
|
|
96acf94521 | ||
|
|
807373d668 | ||
|
|
703e9d0452 |
@@ -1,4 +1,4 @@
|
||||
FROM golang as builder
|
||||
FROM golang AS builder
|
||||
WORKDIR /src
|
||||
RUN git clone https://github.com/tenox7/wrp.git
|
||||
WORKDIR /src/wrp
|
||||
|
||||
6
Dockerfile.local
Normal file
6
Dockerfile.local
Normal file
@@ -0,0 +1,6 @@
|
||||
FROM chromedp/headless-shell
|
||||
ARG TARGETARCH
|
||||
ADD wrp-${TARGETARCH}-linux /wrp
|
||||
ENTRYPOINT ["/wrp"]
|
||||
ENV PATH="/headless-shell:${PATH}"
|
||||
LABEL maintainer="as@tenoware.com"
|
||||
25
Makefile
25
Makefile
@@ -1,23 +1,28 @@
|
||||
all: wrp
|
||||
|
||||
wrp: wrp.go
|
||||
go build wrp.go
|
||||
go build -a
|
||||
|
||||
cross:
|
||||
GOOS=linux GOARCH=amd64 go build -a -o wrp-amd64-linux wrp.go
|
||||
GOOS=freebsd GOARCH=amd64 go build -a -o wrp-amd64-freebsd wrp.go
|
||||
GOOS=openbsd GOARCH=amd64 go build -a -o wrp-amd64-openbsd wrp.go
|
||||
GOOS=darwin GOARCH=amd64 go build -a -o wrp-amd64-macos wrp.go
|
||||
GOOS=darwin GOARCH=arm64 go build -a -o wrp-arm64-macos wrp.go
|
||||
GOOS=windows GOARCH=amd64 go build -a -o wrp-amd64-windows.exe wrp.go
|
||||
GOOS=linux GOARCH=arm go build -a -o wrp-arm-linux wrp.go
|
||||
GOOS=linux GOARCH=arm64 go build -a -o wrp-arm64-linux wrp.go
|
||||
GOOS=linux GOARCH=amd64 go build -a -o wrp-amd64-linux
|
||||
GOOS=freebsd GOARCH=amd64 go build -a -o wrp-amd64-freebsd
|
||||
GOOS=openbsd GOARCH=amd64 go build -a -o wrp-amd64-openbsd
|
||||
GOOS=darwin GOARCH=amd64 go build -a -o wrp-amd64-macos
|
||||
GOOS=darwin GOARCH=arm64 go build -a -o wrp-arm64-macos
|
||||
GOOS=windows GOARCH=amd64 go build -a -o wrp-amd64-windows.exe
|
||||
GOOS=linux GOARCH=arm go build -a -o wrp-arm-linux
|
||||
GOOS=linux GOARCH=arm64 go build -a -o wrp-arm64-linux
|
||||
|
||||
docker-local:
|
||||
docker buildx build --platform linux/amd64,linux/arm64 -t tenox7/wrp:latest --load .
|
||||
GOOS=linux GOARCH=amd64 go build -a -o wrp-amd64-linux
|
||||
GOOS=linux GOARCH=arm64 go build -a -o wrp-arm64-linux
|
||||
docker buildx build --platform linux/amd64,linux/arm64 -t tenox7/wrp:latest -f Dockerfile.local --load .
|
||||
|
||||
docker-push:
|
||||
docker buildx build --platform linux/amd64,linux/arm64 -t tenox7/wrp:latest --push .
|
||||
|
||||
docker-clean:
|
||||
docker buildx prune -a -f
|
||||
|
||||
clean:
|
||||
rm -rf wrp-* wrp
|
||||
|
||||
@@ -67,6 +67,8 @@ WRP supports customizing it's own UI using HTML Template file. Download [wrp.htm
|
||||
|
||||
## Docker
|
||||
|
||||
https://hub.docker.com/r/tenox7/wrp
|
||||
|
||||
```shell
|
||||
$ docker run -d --rm -p 8080:8080 tenox7/wrp:latest
|
||||
```
|
||||
|
||||
6
go.mod
6
go.mod
@@ -5,10 +5,13 @@ go 1.21.5
|
||||
require (
|
||||
github.com/JohannesKaufmann/html-to-markdown v1.6.0
|
||||
github.com/MaxHalford/halfgone v0.0.0-20171017091812-482157b86ccb
|
||||
github.com/breml/rootcerts v0.2.17
|
||||
github.com/chromedp/cdproto v0.0.0-20240519224452-66462be74baa
|
||||
github.com/chromedp/chromedp v0.9.5
|
||||
github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
|
||||
github.com/soniakeys/quant v1.0.0
|
||||
github.com/yuin/goldmark v1.7.2
|
||||
golang.org/x/image v0.18.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -20,7 +23,6 @@ require (
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/yuin/goldmark v1.7.2 // indirect
|
||||
golang.org/x/net v0.25.0 // indirect
|
||||
golang.org/x/sys v0.20.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
||||
9
go.sum
9
go.sum
@@ -6,6 +6,8 @@ github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4
|
||||
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/breml/rootcerts v0.2.17 h1:0/M2BE2Apw0qEJCXDOkaiu7d5Sx5ObNfe1BkImJ4u1I=
|
||||
github.com/breml/rootcerts v0.2.17/go.mod h1:S/PKh+4d1HUn4HQovEB8hPJZO6pUZYrIhmXBhsegfXw=
|
||||
github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
|
||||
github.com/chromedp/cdproto v0.0.0-20240519224452-66462be74baa h1:T3Ho4BWIkoEoMPCj90W2HIPF/k56qk4JWzTs6JUBxVw=
|
||||
github.com/chromedp/cdproto v0.0.0-20240519224452-66462be74baa/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
|
||||
@@ -22,8 +24,6 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
|
||||
github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2 h1:yEt5djSYb4iNtmV9iJGVday+i4e9u6Mrn5iP64HH5QM=
|
||||
github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
@@ -35,6 +35,8 @@ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kUL
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@@ -51,7 +53,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U=
|
||||
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
||||
github.com/yuin/goldmark v1.7.2 h1:NjGd7lO7zrUn/A7eKwn5PEOt4ONYGqpxSEeZuduvgxc=
|
||||
github.com/yuin/goldmark v1.7.2/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
||||
@@ -60,6 +61,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
|
||||
golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
|
||||
221
txt.go
Normal file
221
txt.go
Normal file
@@ -0,0 +1,221 @@
|
||||
package main
|
||||
|
||||
// TODO:
|
||||
// - image type based on form value
|
||||
// - also size and quality
|
||||
// - non overlaping image names atomic.int etc
|
||||
// - garbage collector / delete old images from map
|
||||
// - add referer header
|
||||
// - svg support
|
||||
// - BOG: DomainFromURL always prefixes with http instead of https
|
||||
// reproduces on vsi vms docs
|
||||
// - BUG: markdown table errors
|
||||
// reproduces on hacker news
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
h2m "github.com/JohannesKaufmann/html-to-markdown"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/plugin"
|
||||
"github.com/nfnt/resize"
|
||||
"github.com/yuin/goldmark"
|
||||
"github.com/yuin/goldmark/ast"
|
||||
"github.com/yuin/goldmark/extension"
|
||||
"github.com/yuin/goldmark/parser"
|
||||
"github.com/yuin/goldmark/text"
|
||||
"github.com/yuin/goldmark/util"
|
||||
"golang.org/x/image/webp"
|
||||
)
|
||||
|
||||
var imgStor imageStore
|
||||
|
||||
const imgZpfx = "/imgz/"
|
||||
|
||||
func init() {
|
||||
imgStor.img = make(map[string]imageContainer)
|
||||
// TODO: add garbage collector
|
||||
// think about how to remove old images
|
||||
// if removed from cache how to download them later if a browser goes back?
|
||||
// browser should cache on it's own... but it may request it, what then?
|
||||
}
|
||||
|
||||
type imageContainer struct {
|
||||
data []byte
|
||||
url string
|
||||
added time.Time
|
||||
}
|
||||
|
||||
type imageStore struct {
|
||||
img map[string]imageContainer
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
func (i *imageStore) add(id, url string, img []byte) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
i.img[id] = imageContainer{data: img, url: url, added: time.Now()}
|
||||
}
|
||||
|
||||
func (i *imageStore) get(id string) ([]byte, error) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
img, ok := i.img[id]
|
||||
if !ok {
|
||||
return nil, errors.New("not found")
|
||||
}
|
||||
return img.data, nil
|
||||
}
|
||||
|
||||
func (i *imageStore) del(id string) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
delete(i.img, id)
|
||||
}
|
||||
|
||||
func fetchImage(id, url string) error {
|
||||
log.Printf("Downloading IMGZ URL=%q for ID=%q", url, id)
|
||||
var img []byte
|
||||
var err error
|
||||
switch url[:4] {
|
||||
case "http":
|
||||
r, err := http.Get(url) // TODO: possibly set a header "referer" here
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error downloading %q: %v", url, err)
|
||||
}
|
||||
if r.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("Error %q HTTP Status Code: %v", url, r.StatusCode)
|
||||
}
|
||||
defer r.Body.Close()
|
||||
img, err = io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error reading %q: %v", url, err)
|
||||
}
|
||||
case "data":
|
||||
idx := strings.Index(url, ",")
|
||||
if idx < 1 {
|
||||
return fmt.Errorf("image is embeded but unable to find coma: %q", url)
|
||||
}
|
||||
img, err = base64.StdEncoding.DecodeString(url[idx+1:])
|
||||
if err != nil {
|
||||
return fmt.Errorf("error decoding image from url embed: %q: %v", url, err)
|
||||
}
|
||||
}
|
||||
gif, err := smallGif(img)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error scaling down image: %v", err)
|
||||
}
|
||||
imgStor.add(id, url, gif)
|
||||
return nil
|
||||
}
|
||||
|
||||
type astTransformer struct{}
|
||||
|
||||
func (t *astTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
|
||||
ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
if link, ok := n.(*ast.Link); ok && entering {
|
||||
link.Destination = append([]byte("/?t=txt&url="), link.Destination...)
|
||||
}
|
||||
if img, ok := n.(*ast.Image); ok && entering {
|
||||
// TODO: dynamic extension based on form value
|
||||
id := fmt.Sprintf("txt%05d.gif", rand.Intn(99999)) // BUG: atomic.AddInt64 or something that ever increases - time based?
|
||||
err := fetchImage(id, string(img.Destination)) // TODO: use goroutines with waitgroup
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
n.Parent().RemoveChildren(n)
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
img.Destination = []byte(imgZpfx + id)
|
||||
}
|
||||
return ast.WalkContinue, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (rq *wrpReq) captureMarkdown() {
|
||||
log.Printf("Processing Markdown conversion request for %v", rq.url)
|
||||
// TODO: bug - DomainFromURL always prefixes with http:// instead of https
|
||||
// this causes issues on some websites, fix or write a smarter DomainFromURL
|
||||
c := h2m.NewConverter(h2m.DomainFromURL(rq.url), true, nil)
|
||||
c.Use(plugin.GitHubFlavored())
|
||||
md, err := c.ConvertURL(rq.url) // We could also get inner html from chromedp
|
||||
if err != nil {
|
||||
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
log.Printf("Got %v bytes md from %v", len(md), rq.url)
|
||||
t := &astTransformer{}
|
||||
gm := goldmark.New(
|
||||
goldmark.WithExtensions(extension.GFM),
|
||||
goldmark.WithParserOptions(parser.WithASTTransformers(util.Prioritized(t, 100))),
|
||||
)
|
||||
var ht bytes.Buffer
|
||||
err = gm.Convert([]byte(md), &ht)
|
||||
if err != nil {
|
||||
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
log.Printf("Rendered %v bytes html for %v", len(ht.String()), rq.url)
|
||||
rq.printHTML(printParams{
|
||||
text: string(asciify([]byte(ht.String()))),
|
||||
bgColor: "#FFFFFF",
|
||||
})
|
||||
}
|
||||
|
||||
func imgServerZ(w http.ResponseWriter, r *http.Request) {
|
||||
log.Printf("%s IMGZ Request for %s", r.RemoteAddr, r.URL.Path)
|
||||
id := strings.Replace(r.URL.Path, imgZpfx, "", 1)
|
||||
img, err := imgStor.get(id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
log.Printf("%s IMGZ error for %s: %v", r.RemoteAddr, r.URL.Path, err)
|
||||
return
|
||||
}
|
||||
imgStor.del(id)
|
||||
w.Header().Set("Content-Type", http.DetectContentType(img))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(img)))
|
||||
w.Write(img)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
// TODO set JPG/GIF/PNG type based on form...
|
||||
func smallGif(src []byte) ([]byte, error) {
|
||||
t := http.DetectContentType(src)
|
||||
var err error
|
||||
var img image.Image
|
||||
switch t {
|
||||
case "image/png":
|
||||
img, err = png.Decode(bytes.NewReader(src))
|
||||
case "image/gif":
|
||||
img, err = gif.Decode(bytes.NewReader(src))
|
||||
case "image/jpeg":
|
||||
img, err = jpeg.Decode(bytes.NewReader(src))
|
||||
case "image/webp":
|
||||
img, err = webp.Decode(bytes.NewReader(src))
|
||||
default: // TODO: also add svg
|
||||
err = errors.New("unknown content type: " + t)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("image decode problem: %v", err)
|
||||
}
|
||||
img = resize.Thumbnail(uint(*txtImgSize), uint(*txtImgSize), img, resize.NearestNeighbor)
|
||||
var gifBuf bytes.Buffer
|
||||
err = gif.Encode(&gifBuf, gifPalette(img, 216), &gif.Options{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gif encode problem: %v", err)
|
||||
}
|
||||
return gifBuf.Bytes(), nil
|
||||
}
|
||||
83
wrp.go
83
wrp.go
@@ -35,30 +35,24 @@ import (
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
h2m "github.com/JohannesKaufmann/html-to-markdown"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/plugin"
|
||||
"github.com/MaxHalford/halfgone"
|
||||
_ "github.com/breml/rootcerts"
|
||||
"github.com/chromedp/cdproto/css"
|
||||
"github.com/chromedp/cdproto/emulation"
|
||||
"github.com/chromedp/cdproto/input"
|
||||
"github.com/chromedp/cdproto/page"
|
||||
"github.com/chromedp/chromedp"
|
||||
"github.com/soniakeys/quant/median"
|
||||
"github.com/yuin/goldmark"
|
||||
"github.com/yuin/goldmark/ast"
|
||||
"github.com/yuin/goldmark/extension"
|
||||
"github.com/yuin/goldmark/parser"
|
||||
"github.com/yuin/goldmark/text"
|
||||
"github.com/yuin/goldmark/util"
|
||||
)
|
||||
|
||||
const version = "4.6.3"
|
||||
const version = "4.7.2"
|
||||
|
||||
var (
|
||||
addr = flag.String("l", ":8080", "Listen address:port, default :8080")
|
||||
headless = flag.Bool("h", true, "Headless mode / hide browser window (default true)")
|
||||
noDel = flag.Bool("n", false, "Do not free maps and images after use")
|
||||
defType = flag.String("t", "gif", "Image type: png|gif|jpg|txt")
|
||||
txtImgSize = flag.Int("ts", 200, "txt mode image size") // make it default, this should come from the from
|
||||
jpgQual = flag.Int("q", 80, "Jpeg image quality, default 80%")
|
||||
fgeom = flag.String("g", "1152x600x216", "Geometry: width x height x colors, height can be 0 for unlimited")
|
||||
htmFnam = flag.String("ui", "wrp.html", "HTML template file for the UI")
|
||||
@@ -170,6 +164,9 @@ func (rq *wrpReq) printHTML(p printParams) {
|
||||
rq.w.Header().Set("Expires", "-1")
|
||||
rq.w.Header().Set("Pragma", "no-cache")
|
||||
rq.w.Header().Set("Content-Type", "text/html")
|
||||
if p.bgColor == "" {
|
||||
p.bgColor = "#FFFFFF"
|
||||
}
|
||||
data := uiData{
|
||||
Version: version,
|
||||
URL: rq.url,
|
||||
@@ -320,6 +317,7 @@ func gifPalette(i image.Image, n int64) image.Image {
|
||||
func (rq *wrpReq) captureImage() {
|
||||
var styles []*css.ComputedStyleProperty
|
||||
var r, g, b int
|
||||
var bgColorSet bool
|
||||
var h int64
|
||||
var pngCap []byte
|
||||
chromedp.Run(ctx,
|
||||
@@ -336,9 +334,17 @@ func (rq *wrpReq) captureImage() {
|
||||
)
|
||||
log.Printf("%s Landed on: %s, Height: %v\n", rq.r.RemoteAddr, rq.url, h)
|
||||
for _, style := range styles {
|
||||
if style.Name == "background-color" {
|
||||
fmt.Sscanf(style.Value, "rgb(%d,%d,%d)", &r, &g, &b)
|
||||
if style.Name != "background-color" {
|
||||
continue
|
||||
}
|
||||
fmt.Sscanf(style.Value, "rgb(%d,%d,%d)", &r, &g, &b)
|
||||
bgColorSet = true
|
||||
break
|
||||
}
|
||||
if !bgColorSet {
|
||||
r = 255
|
||||
g = 255
|
||||
b = 255
|
||||
}
|
||||
height := int64(float64(rq.height) / rq.zoom)
|
||||
if rq.height == 0 && h > 0 {
|
||||
@@ -430,51 +436,6 @@ func asciify(s []byte) []byte {
|
||||
return a
|
||||
}
|
||||
|
||||
type astTransformer struct{}
|
||||
|
||||
func (t *astTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
|
||||
ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
if link, ok := n.(*ast.Link); ok && entering {
|
||||
link.Destination = append([]byte("?t=txt&url="), link.Destination...)
|
||||
}
|
||||
if _, ok := n.(*ast.Image); ok && entering {
|
||||
// TODO: perhaps instead of deleting images convert them to links
|
||||
// smaller images or ascii? https://github.com/TheZoraiz/ascii-image-converter
|
||||
n.Parent().RemoveChildren(n)
|
||||
}
|
||||
return ast.WalkContinue, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (rq *wrpReq) captureMarkdown() {
|
||||
log.Printf("Processing Markdown conversion request for %v", rq.url)
|
||||
// TODO: bug - DomainFromURL always prefixes with http:// instead of https
|
||||
// this causes issues on some websites, fix or write a smarter DomainFromURL
|
||||
c := h2m.NewConverter(h2m.DomainFromURL(rq.url), true, nil)
|
||||
c.Use(plugin.GitHubFlavored())
|
||||
md, err := c.ConvertURL(rq.url) // We could also get inner html from chromedp
|
||||
if err != nil {
|
||||
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
log.Printf("Got %v bytes md from %v", len(md), rq.url)
|
||||
gm := goldmark.New(
|
||||
goldmark.WithExtensions(extension.GFM),
|
||||
goldmark.WithParserOptions(parser.WithASTTransformers(util.Prioritized(&astTransformer{}, 100))),
|
||||
)
|
||||
var ht bytes.Buffer
|
||||
err = gm.Convert([]byte(md), &ht)
|
||||
if err != nil {
|
||||
http.Error(rq.w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
log.Printf("Rendered %v bytes html for %v", len(ht.String()), rq.url)
|
||||
rq.printHTML(printParams{
|
||||
text: string(asciify([]byte(ht.String()))),
|
||||
bgColor: "#FFFFFF",
|
||||
})
|
||||
}
|
||||
|
||||
// Process HTTP requests to WRP '/' url
|
||||
func pageServer(w http.ResponseWriter, r *http.Request) {
|
||||
log.Printf("%s Page Request for %s [%+v]\n", r.RemoteAddr, r.URL.Path, r.URL.RawQuery)
|
||||
@@ -537,11 +498,11 @@ func imgServer(w http.ResponseWriter, r *http.Request) {
|
||||
defer delete(img, r.URL.Path)
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(r.URL.Path, ".gif"):
|
||||
case strings.HasSuffix(r.URL.Path, ".gif"):
|
||||
w.Header().Set("Content-Type", "image/gif")
|
||||
case strings.HasPrefix(r.URL.Path, ".png"):
|
||||
case strings.HasSuffix(r.URL.Path, ".png"):
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
case strings.HasPrefix(r.URL.Path, ".jpg"):
|
||||
case strings.HasSuffix(r.URL.Path, ".jpg"):
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(imgBuf.Bytes())))
|
||||
@@ -631,6 +592,7 @@ func main() {
|
||||
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||
flag.Parse()
|
||||
log.Printf("Web Rendering Proxy Version %s (%v)\n", version, runtime.GOARCH)
|
||||
log.Printf("Using embedded ca-certs from github.com/breml/rootcerts")
|
||||
log.Printf("Args: %q", os.Args)
|
||||
if len(os.Getenv("PORT")) > 0 {
|
||||
*addr = ":" + os.Getenv(("PORT"))
|
||||
@@ -655,8 +617,6 @@ func main() {
|
||||
ctx, cncl = chromedp.NewContext(actx)
|
||||
defer cncl()
|
||||
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
|
||||
c := make(chan os.Signal)
|
||||
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
@@ -671,6 +631,7 @@ func main() {
|
||||
http.HandleFunc("/", pageServer)
|
||||
http.HandleFunc("/map/", mapServer)
|
||||
http.HandleFunc("/img/", imgServer)
|
||||
http.HandleFunc(imgZpfx, imgServerZ)
|
||||
http.HandleFunc("/shutdown/", haltServer)
|
||||
http.HandleFunc("/favicon.ico", http.NotFound)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user