From a3152ee4409c95cb1cc1799137930862863286c6 Mon Sep 17 00:00:00 2001 From: Tordarus Date: Wed, 10 Sep 2025 11:03:47 +0200 Subject: [PATCH] resolve external chapter links --- api.go | 2 ++ curl.go | 42 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 7 +------ go.sum | 27 ++------------------------- 4 files changed, 47 insertions(+), 31 deletions(-) diff --git a/api.go b/api.go index 909f946..bd81b6d 100644 --- a/api.go +++ b/api.go @@ -181,6 +181,8 @@ func (api *Api) getChapterEntriesByPageIndex(novelID NovelID, pageIndex int) ([] return } + link = api.ResolveExternalLink(link) + entries = append(entries, NovelChapterEntry{ NovelID: novelID, ID: chapterID, diff --git a/curl.go b/curl.go index b9d1a47..b10f42e 100644 --- a/curl.go +++ b/curl.go @@ -4,8 +4,13 @@ import ( "compress/gzip" "context" "fmt" + "net/http" + "net/url" "os/exec" + "path" + "strings" + "git.tordarus.net/tordarus/ezhttp" "github.com/PuerkitoBio/goquery" ) @@ -68,3 +73,40 @@ func (api *Api) Get(url string) (*goquery.Document, error) { return goquery.NewDocumentFromReader(r) } + +func (api *Api) ResolveExternalLink(link string) string { + if api.Cookie == "" { + return link + } + + parsedLink, err := url.Parse(link) + if err != nil { + return link + } + + if !strings.HasPrefix(path.Clean(parsedLink.Path), "/extnu/") { + return link + } + + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + req := ezhttp.Request( + ezhttp.URL(link), + ezhttp.AddHeaders( + "User-Agent", api.UserAgent, + "Cookie", api.Cookie, + ), + ) + + resp, err := client.Do(req) + if err != nil { + return link + } + defer resp.Body.Close() + + return resp.Header.Get("Location") +} diff --git a/go.mod b/go.mod index f8fe5c8..2990a61 100644 --- a/go.mod +++ b/go.mod @@ -6,16 +6,11 @@ toolchain go1.24.6 require ( git.tordarus.net/tordarus/adverr/v2 v2.0.2 + git.tordarus.net/tordarus/ezhttp v0.0.9 github.com/PuerkitoBio/goquery v1.8.1 ) require ( - git.tordarus.net/tordarus/gmath v0.0.7 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect - github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect - github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect - github.com/go-shiori/go-readability v0.0.0-20250217085726-9f5bf5ca7612 // indirect - github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect golang.org/x/net v0.43.0 // indirect - golang.org/x/text v0.28.0 // indirect ) diff --git a/go.sum b/go.sum index 4143477..8a6d6d6 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,13 @@ git.tordarus.net/tordarus/adverr/v2 v2.0.2 h1:7nvNjMMjtGPq0EY6duMiv+seJ7MacNvKSBmckHl6Erg= git.tordarus.net/tordarus/adverr/v2 v2.0.2/go.mod h1:gCC46KsWosZJh7MVNDEU99hKQoxEWZgHITDHtmFwwiQ= -git.tordarus.net/tordarus/gmath v0.0.7 h1:tR48idt9AUL0r556ww3ZxByTKJEr6NWCTlhl2ihzYxQ= -git.tordarus.net/tordarus/gmath v0.0.7/go.mod h1:mO7aPlvNrGVE9UFXEuuACjZgMDsM63l3OcQy6xSQnoE= -git.tordarus.net/tordarus/slices v0.0.15 h1:qxKS+7BCZ/LzQbRCvdDFoLQXaJZ2C0GfVju/kPt1r3g= -git.tordarus.net/tordarus/slices v0.0.15/go.mod h1:eJBw6pSDNivPI0l4e0sGKUJzou/lbzHflXdAUzp1g4o= +git.tordarus.net/tordarus/ezhttp v0.0.9 h1:YwdQ4YcJwvpMw5CX5NcCEM23XQL+WCz5nWuc2dzX/84= +git.tordarus.net/tordarus/ezhttp v0.0.9/go.mod h1:Zq9o0Hibny61GqSCwJHa0PfGjVoUFv/zt2PjiQHXvmY= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= -github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= -github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= -github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= -github.com/go-shiori/go-readability v0.0.0-20250217085726-9f5bf5ca7612 h1:BYLNYdZaepitbZreRIa9xeCQZocWmy/wj4cGIH0qyw0= -github.com/go-shiori/go-readability v0.0.0-20250217085726-9f5bf5ca7612/go.mod h1:wgqthQa8SAYs0yyljVeCOQlZ027VW5CmLsbi9jWC08c= -github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= -github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -43,7 +25,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= @@ -90,8 +71,6 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -99,5 +78,3 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=