Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues 7 Pull requests Projects Releases Packages Wiki Activity Actions

i need better way to cache images

Browse source
This commit is contained in:
partisan 2024-09-12 22:11:39 +02:00
parent 61af6fc453
commit d107d41d72
8 changed files with 59 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
go.mod
View file

@ -23,6 +23,9 @@ require (
)
require (
github.com/chai2010/webp v1.1.1 // indirect
github.com/disintegration/imaging v1.6.2 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
golang.org/x/image v0.20.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)

8
go.sum
View file

@ -2,12 +2,16 @@ github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VP
github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 h1:XYUCaZrW8ckGWlCRJKCSoh/iFwlpX316a8yY9IFEzv8=
github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93rg=
github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y=
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
@ -31,6 +35,10 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.20.0 h1:7cVCUjQwfL18gyBJOmYvptfSHS8Fb3YUDtfLIZ7Nbpw=
golang.org/x/image v0.20.0/go.mod h1:0a88To4CYVBAHp5FXJm8o7QbUl37Vd85ply1vyD8auM=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

21
imageproxy.go
View file

@ -1,7 +1,6 @@
package main
import (
"fmt"
"io"
"net/http"
)
@ -14,20 +13,12 @@ func handleImageProxy(w http.ResponseWriter, r *http.Request) {
return
}
// Try to fetch the image from Bing first
bingURL := fmt.Sprintf("https://tse.mm.bing.net/th?q=%s", imageURL)
resp, err := http.Get(bingURL)
if err != nil || resp.StatusCode != http.StatusOK {
// If fetching from Bing fails, attempt to fetch from the original image URL
printWarn("Error fetching image from Bing, trying original URL.")
// Attempt to fetch the image directly
resp, err = http.Get(imageURL)
if err != nil {
printWarn("Error fetching image: %v", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
// Fetch the image from the external URL
resp, err := http.Get(imageURL)
if err != nil {
printWarn("Error fetching image: %v", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
defer resp.Body.Close()

9
images-bing.go
View file

@ -12,6 +12,7 @@ import (
"github.com/PuerkitoBio/goquery"
)
// PerformBingImageSearch performs a Bing image search and returns the results.
func PerformBingImageSearch(query, safe, lang string, page int) ([]ImageSearchResult, time.Duration, error) {
startTime := time.Now()
@ -66,14 +67,16 @@ func PerformBingImageSearch(query, safe, lang string, page int) ([]ImageSearchRe
if err := json.Unmarshal([]byte(metadata), &data); err == nil {
mediaURL, ok := data["murl"].(string)
if ok {
// Apply the image proxy
proxiedURL := "/imgproxy?url=" + mediaURL
results = append(results, ImageSearchResult{
Thumbnail: imgSrc,
Title: strings.TrimSpace(title),
Media: mediaURL,
Source: mediaURL,
ThumbProxy: proxiedURL, // Use the proxied URL
Width: width,
Height: height,
Source: mediaURL,
ThumbProxy: imgSrc,
})
}
}
@ -89,6 +92,7 @@ func PerformBingImageSearch(query, safe, lang string, page int) ([]ImageSearchRe
return results, duration, nil
}
// buildBingSearchURL constructs the search URL for Bing Image Search
func buildBingSearchURL(query string, page int) string {
baseURL := "https://www.bing.com/images/search"
params := url.Values{}
@ -99,6 +103,7 @@ func buildBingSearchURL(query string, page int) string {
return baseURL + "?" + params.Encode()
}
// Example usage in main (commented out for clarity)
// func main() {
// results, duration, err := PerformBingImageSearch("kittens", "false", "en", 1)
// if err != nil {

2
images-deviantart.go
View file

@ -156,7 +156,7 @@ func PerformDeviantArtImageSearch(query, safe, lang string, page int) ([]ImageSe
Width: 0,
Height: 0,
Source: resultURL,
ThumbProxy: imgSrc,
ThumbProxy: "/imgproxy?url=" + imgSrc,
}
}
}(imgSrc, resultURL, title)

41
images-quant.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"net/url"
"sync"
"time"
)
@ -141,19 +142,37 @@ func PerformQwantImageSearch(query, safe, lang string, page int) ([]ImageSearchR
return nil, 0, fmt.Errorf("decoding response: %v", err)
}
var results []ImageSearchResult
for _, item := range apiResp.Data.Result.Items {
results = append(results, ImageSearchResult{
Thumbnail: item.Thumbnail,
Title: item.Title,
Media: item.Media,
Source: item.Url,
ThumbProxy: "/img_proxy?url=" + url.QueryEscape(item.Media), // New proxy not exactly working as intended
Width: item.Width,
Height: item.Height,
})
var wg sync.WaitGroup
results := make([]ImageSearchResult, len(apiResp.Data.Result.Items))
for i, item := range apiResp.Data.Result.Items {
wg.Add(1)
go func(i int, item struct {
Media string `json:"media"`
Thumbnail string `json:"thumbnail"`
Title string `json:"title"`
Url string `json:"url"`
Width int `json:"width"`
Height int `json:"height"`
}) {
defer wg.Done()
// Populate the result
results[i] = ImageSearchResult{
Thumbnail: item.Thumbnail,
Title: item.Title,
Media: item.Media,
Source: item.Url,
ThumbProxy: "/imgproxy?url=" + item.Media,
Width: item.Width,
Height: item.Height,
}
}(i, item)
}
// Wait for all goroutines to complete
wg.Wait()
duration := time.Since(startTime) // Calculate the duration
return results, duration, nil

6
images.go
View file

@ -13,9 +13,9 @@ var imageSearchEngines []SearchEngine
func init() {
imageSearchEngines = []SearchEngine{
{Name: "Qwant", Func: wrapImageSearchFunc(PerformQwantImageSearch), Weight: 1},
{Name: "DeviantArt", Func: wrapImageSearchFunc(PerformDeviantArtImageSearch), Weight: 2},
{Name: "Bing", Func: wrapImageSearchFunc(PerformBingImageSearch), Weight: 2}, // Bing sometimes returns with low amount of images, this leads to danamica page loading not working
{Name: "Imgur", Func: wrapImageSearchFunc(PerformImgurImageSearch), Weight: 3},
{Name: "Bing", Func: wrapImageSearchFunc(PerformBingImageSearch), Weight: 2},
{Name: "DeviantArt", Func: wrapImageSearchFunc(PerformDeviantArtImageSearch), Weight: 3},
//{Name: "Imgur", Func: wrapImageSearchFunc(PerformImgurImageSearch), Weight: 4}, // Image proxy not working
}
}

2
main.go
View file

@ -183,7 +183,7 @@ func runServer() {
http.HandleFunc("/", handleSearch)
http.HandleFunc("/search", handleSearch)
http.HandleFunc("/suggestions", handleSuggestions)
http.HandleFunc("/img_proxy", handleImageProxy)
http.HandleFunc("/imgproxy", handleImageProxy)
http.HandleFunc("/node", handleNodeRequest)
http.HandleFunc("/settings", handleSettings)
http.HandleFunc("/save-settings", handleSaveSettings)