Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues 7 Pull requests Projects Releases Packages Wiki Activity Actions

hopefully fixed dynamic image loading

Browse source
This commit is contained in:
partisan 2024-11-13 16:59:42 +01:00
parent 3861fdc81c
commit ccba37021a
5 changed files with 250 additions and 269 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

169
images.go
View file

@ -19,11 +19,24 @@ func init() {
}
}
func handleImageSearch(w http.ResponseWriter, settings UserSettings, query string, page int) {
func handleImageSearch(w http.ResponseWriter, r *http.Request, settings UserSettings, query string, page int) {
startTime := time.Now()
cacheKey := CacheKey{Query: query, Page: page, Safe: settings.SafeSearch == "active", Lang: settings.SearchLanguage, Type: "image"}
combinedResults := getImageResultsFromCacheOrFetch(cacheKey, query, settings.SafeSearch, settings.SearchLanguage, page)
cacheKey := CacheKey{
Query: query,
Page: page,
Safe: settings.SafeSearch == "active",
Lang: settings.SearchLanguage,
Type: "image",
}
// Check if JavaScript is disabled
jsDisabled := r.URL.Query().Get("js_disabled") == "true"
// Determine if we should cache images synchronously
synchronous := jsDisabled
combinedResults := getImageResultsFromCacheOrFetch(cacheKey, query, settings.SafeSearch, settings.SearchLanguage, page, synchronous)
elapsedTime := time.Since(startTime)
@ -31,7 +44,7 @@ func handleImageSearch(w http.ResponseWriter, settings UserSettings, query strin
data := map[string]interface{}{
"Results": combinedResults,
"Query": query,
"Fetched": fmt.Sprintf("%.2f %s", elapsedTime.Seconds(), Translate("seconds")), // Time for fetching
"Fetched": fmt.Sprintf("%.2f %s", elapsedTime.Seconds(), Translate("seconds")),
"Page": page,
"HasPrevPage": page > 1,
"HasNextPage": len(combinedResults) >= 50,
@ -41,14 +54,15 @@ func handleImageSearch(w http.ResponseWriter, settings UserSettings, query strin
"Theme": settings.Theme,
"Safe": settings.SafeSearch,
"IsThemeDark": settings.IsThemeDark,
"HardCacheEnabled": config.HardCacheDuration == 0,
"HardCacheEnabled": config.HardCacheEnabled,
"JsDisabled": jsDisabled,
}
// Render the template without measuring the time
// Render the full page
renderTemplate(w, "images.html", data)
}
func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page int) []ImageSearchResult {
func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page int, synchronous bool) []ImageSearchResult {
cacheChan := make(chan []SearchResult)
var combinedResults []ImageSearchResult
@ -66,7 +80,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
select {
case results := <-cacheChan:
if results == nil {
combinedResults = fetchImageResults(query, safe, lang, page)
combinedResults = fetchImageResults(query, safe, lang, page, synchronous)
if len(combinedResults) > 0 {
combinedResults = filterValidImages(combinedResults)
resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
@ -77,7 +91,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
}
case <-time.After(2 * time.Second):
printInfo("Cache check timeout")
combinedResults = fetchImageResults(query, safe, lang, page)
combinedResults = fetchImageResults(query, safe, lang, page, synchronous)
if len(combinedResults) > 0 {
combinedResults = filterValidImages(combinedResults)
resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
@ -87,7 +101,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
return combinedResults
}
func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
func fetchImageResults(query, safe, lang string, page int, synchronous bool) []ImageSearchResult {
var results []ImageSearchResult
engineCount := len(imageSearchEngines)
safeBool := safe == "active"
@ -99,9 +113,6 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
// Calculate the specific page number for the selected engine
enginePage := (page-1)/engineCount + 1
// Debug print to verify the selected engine and page
printInfo("Fetching image results for overall page %d using engine: %s (engine page %d)", page, engine.Name, enginePage)
// Fetch results from the selected engine
searchResults, _, err := engine.Func(query, safe, lang, enginePage)
if err != nil {
@ -118,19 +129,26 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
imageResult.ID = hash
imageResult.ProxyFull = "/image_cache/" + filename
// Start caching and validation in the background
go func(imgResult ImageSearchResult, originalURL, filename string) {
_, success, err := cacheImage(originalURL, filename, imgResult.ID)
if err != nil {
printWarn("Failed to cache image %s: %v", originalURL, err)
if synchronous {
// Synchronously cache the image
_, success, err := cacheImage(imageResult.Full, filename, imageResult.ID)
if err != nil || !success {
printWarn("Failed to cache image %s: %v", imageResult.Full, err)
// Fallback to proxy URL
imageResult.ProxyFull = "/imgproxy?url=" + imageResult.Full
}
if !success {
removeImageResultFromCache(query, page, safeBool, lang, imgResult.ID)
}
}(imageResult, imageResult.Full, filename)
} else {
// Start caching and validation in the background
go func(imgResult ImageSearchResult, originalURL, filename string) {
_, success, err := cacheImage(originalURL, filename, imgResult.ID)
if err != nil || !success {
printWarn("Failed to cache image %s: %v", originalURL, err)
removeImageResultFromCache(query, page, safeBool, lang, imgResult.ID)
}
}(imageResult, imageResult.Full, filename)
}
} else {
// Use proxied URLs when hard cache is disabled
imageResult.ProxyThumb = "/imgproxy?url=" + imageResult.Thumb
imageResult.ProxyFull = "/imgproxy?url=" + imageResult.Full
}
results = append(results, imageResult)
@ -161,16 +179,30 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
imageResult.ID = hash
imageResult.ProxyFull = "/image_cache/" + filename
// Start caching and validation in the background
go func(imgResult ImageSearchResult, originalURL, filename string) {
_, success, err := cacheImage(originalURL, filename, imgResult.ID)
if synchronous {
// Synchronously cache the image
_, success, err := cacheImage(imageResult.Full, filename, imageResult.ID)
if err != nil {
printWarn("Failed to cache image %s: %v", originalURL, err)
printWarn("Failed to cache image %s: %v", imageResult.Full, err)
// Skip this image
continue
}
if !success {
removeImageResultFromCache(query, page, safeBool, lang, imgResult.ID)
// Skip this image
continue
}
}(imageResult, imageResult.Full, filename)
} else {
// Start caching and validation in the background
go func(imgResult ImageSearchResult, originalURL, filename string) {
_, success, err := cacheImage(originalURL, filename, imgResult.ID)
if err != nil {
printWarn("Failed to cache image %s: %v", originalURL, err)
}
if !success {
removeImageResultFromCache(query, page, safeBool, lang, imgResult.ID)
}
}(imageResult, imageResult.Full, filename)
}
} else {
// Use proxied URLs when hard cache is disabled
imageResult.ProxyThumb = "/imgproxy?url=" + imageResult.Thumb
@ -204,82 +236,3 @@ func wrapImageSearchFunc(f func(string, string, string, int) ([]ImageSearchResul
return searchResults, duration, nil
}
}
// func isValidImageURL(imageURL string) bool {
// client := &http.Client{
// Timeout: 10 * time.Second,
// Transport: &http.Transport{
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
// },
// }
// req, err := http.NewRequest("GET", imageURL, nil)
// if err != nil {
// return false
// }
// // Set headers to mimic a real browser
// req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "+
// "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36")
// req.Header.Set("Accept", "image/webp,image/*,*/*;q=0.8")
// req.Header.Set("Accept-Language", "en-US,en;q=0.9")
// req.Header.Set("Referer", imageURL) // Some servers require a referer
// resp, err := client.Do(req)
// if err != nil {
// return false
// }
// defer resp.Body.Close()
// if resp.StatusCode < 200 || resp.StatusCode >= 400 {
// return false
// }
// // Limit the amount of data read to 10KB
// limitedReader := io.LimitReader(resp.Body, 10240) // 10KB
// // Attempt to decode image configuration
// _, _, err = image.DecodeConfig(limitedReader)
// if err != nil {
// return false
// }
// return true
// }
// // This function can be used alternatively to isValidImageURL(), Its slower but reliable
// func isImageAccessible(imageURL string) bool {
// client := &http.Client{
// Timeout: 5 * time.Second,
// CheckRedirect: func(req *http.Request, via []*http.Request) error {
// if len(via) >= 10 {
// return http.ErrUseLastResponse
// }
// return nil
// },
// }
// resp, err := client.Get(imageURL)
// if err != nil {
// return false
// }
// defer resp.Body.Close()
// if resp.StatusCode < 200 || resp.StatusCode >= 400 {
// return false
// }
// // Read the entire image data
// data, err := io.ReadAll(resp.Body)
// if err != nil {
// return false
// }
// // Try to decode the image
// _, _, err = image.Decode(bytes.NewReader(data))
// if err != nil {
// return false
// }
// return true
// }

2
main.go
View file

@ -161,7 +161,7 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
// Handle different search types
switch searchType {
case "image":
handleImageSearch(w, settings, query, page)
handleImageSearch(w, r, settings, query, page)
case "video":
handleVideoSearch(w, settings, query, page)
case "map":

3
node-handle-search.go
View file

@ -63,8 +63,7 @@ func handleSearchImageMessage(msg Message) {
}
log.Printf("Received search-image request. ResponseAddr: %s", searchParams.ResponseAddr)
results := fetchImageResults(searchParams.Query, searchParams.Safe, searchParams.Lang, searchParams.Page)
results := fetchImageResults(searchParams.Query, searchParams.Safe, searchParams.Lang, searchParams.Page, true)
resultsJSON, err := json.Marshal(results)
if err != nil {
log.Printf("Error marshalling search results: %v", err)

323
templates/images.html
View file

@ -152,6 +152,9 @@
<button name="t" value="file" class="clickable">{{ translate "torrents" }}</button>
</div>
</div>
{{ if not .JsDisabled }}
<input type="hidden" name="js_enabled" value="true">
{{ end }}
</form>
<form class="results_settings" action="/search" method="get">
<input type="hidden" name="q" value="{{ .Query }}">
@ -168,48 +171,60 @@
</form>
<div class="search-results" id="results">
<!-- Results go here -->
{{ if .Results }}
<div class="images images_viewer_hidden">
<!-- Images Grid -->
{{ range $index, $result := .Results }}
<div class="image">
<img
src="/static/images/placeholder.svg"
data-id="{{ $result.ID }}"
alt="{{ .Title }}"
data-full="{{ .ProxyFull }}"
data-proxy-full="{{ .ProxyThumb }}"
class="clickable"
/>
<div class="resolution">{{ .Width }} × {{ .Height }}</div>
<div class="details">
<span class="img_title clickable">{{ .Title }}</span>
<a href="{{ .Source }}" target="_blank" class="img_source">{{ translate "source" }}</a>
</div>
<!-- Results go here -->
{{ if .Results }}
<div class="images images_viewer_hidden">
<!-- Images Grid -->
{{ range $index, $result := .Results }}
<div class="image">
{{ if $.HardCacheEnabled }}
{{ if $.JsDisabled }}
<!-- JavaScript is disabled; serve actual images -->
<img src="{{ $result.ProxyFull }}" alt="{{ $result.Title }}" class="clickable" />
{{ else }}
<!-- JavaScript is enabled; use placeholders -->
<img
src="/static/images/placeholder.svg"
data-id="{{ $result.ID }}"
data-full="{{ $result.ProxyFull }}"
data-proxy-full="{{ $result.ProxyThumb }}"
alt="{{ $result.Title }}"
class="clickable"
/>
{{ end }}
{{ else }}
<!-- HardCacheEnabled is false; serve images directly -->
<img src="{{ $result.ProxyFull }}" alt="{{ $result.Title }}" class="clickable" />
{{ end }}
<div class="resolution">{{ $result.Width }} × {{ $result.Height }}</div>
<div class="details">
<span class="img_title clickable">{{ $result.Title }}</span>
<a href="{{ $result.Source }}" target="_blank" class="img_source">{{ translate "source" }}</a>
</div>
</div>
{{ end }}
</div>
<!-- Nav buttons -->
<noscript>
<form action="/search" id="prev-next-form" class="results-search-container" method="GET" autocomplete="off">
<!-- Existing form fields -->
<input type="hidden" name="js_disabled" value="true">
</form>
<!-- Nav buttons -->
<div class="pagination">
{{ if .HasPrevPage }}
<a href="/search?q={{ .Query }}&t=image&p={{ sub .Page 1 }}&js_disabled=true">{{ translate "previous" }}</a>
{{ end }}
{{ if .HasNextPage }}
<a href="/search?q={{ .Query }}&t=image&p={{ add .Page 1 }}&js_disabled=true">{{ translate "next" }}</a>
{{ end }}
</div>
<noscript>
<div class="prev-next prev-img">
<form action="/search" method="get">
<input type="hidden" name="q" value="{{ .Query }}">
<input type="hidden" name="t" value="image">
{{ if .HasPrevPage }}
<button type="submit" name="p" value="{{ sub .Page 1 }}">{{ translate "previous" }}</button>
{{ end }}
{{ if .HasNextPage }}
<button type="submit" name="p" value="{{ add .Page 1 }}">{{ translate "next" }}</button>
{{ end }}
</form>
</div>
</noscript>
{{ else if .NoResults }}
<div class="no-results">{{ translate "no_results" .Query }}</div>
{{ else }}
<div class="no-more-results">{{ translate "no_more_results" }}</div>
{{ end }}
</noscript>
{{ else if .NoResults }}
<div class="no-results">{{ translate "no_results" .Query }}</div>
{{ else }}
<div class="no-more-results">{{ translate "no_more_results" }}</div>
{{ end }}
</div>
<div class="message-bottom-left" id="message-bottom-left">
<span>{{ translate "searching_for_new_results" }}</span>
@ -218,141 +233,133 @@
<div id="image-viewer-overlay" style="display: none;"></div>
<div id="template-data" data-page="{{ .Page }}" data-query="{{ .Query }}" data-type="image" data-hard-cache-enabled="{{ .HardCacheEnabled }}"></div>
<script defer src="/static/js/dynamicscrolling.js"></script>
<script defer src="/static/js/autocomplete.js"></script>
<script defer src="/static/js/imagetitletrim.js"></script>
<script defer src="/static/js/imageviewer.js"></script>
<!-- JavaScript to Load Images -->
<!-- JavaScript to Load Images and Dynamic loading of the page -->
<script>
document.addEventListener("DOMContentLoaded", function() {
const templateData = document.getElementById('template-data');
const hardCacheEnabled = templateData.getAttribute('data-hard-cache-enabled') === 'true';
if (hardCacheEnabled) {
// Immediate loading of images
const images = document.querySelectorAll("img[data-id]");
images.forEach((img) => {
// Use the ProxyFull URL directly
img.src = img.dataset.proxyFull;
img.onload = function() {
img.classList.add('loaded');
};
img.onerror = function() {
console.error('Failed to load image:', img.dataset.proxyFull);
};
});
return;
}
let imageMap = {}; // Map of image IDs to img elements
let loadedImageIDs = new Set(); // Keep track of loaded image IDs
let pollingInterval = 2000; // Polling interval in milliseconds
function initializeImages() {
const images = document.querySelectorAll("img[data-id]");
images.forEach((img) => {
const id = img.dataset.id;
if (!imageMap[id]) {
imageMap[id] = img;
}
});
}
// Initialize with images present at page load
initializeImages();
// Set up MutationObserver to detect new images added to the DOM
const observer = new MutationObserver((mutationsList) => {
for (let mutation of mutationsList) {
if (mutation.type === 'childList') {
mutation.addedNodes.forEach((node) => {
if (node.nodeType === Node.ELEMENT_NODE) {
if (node.matches && node.matches('img[data-id]')) {
const img = node;
const id = img.dataset.id;
if (!imageMap[id]) {
imageMap[id] = img;
console.log('New image added:', id);
}
} else {
// Check for nested images within added nodes
const nestedImages = node.querySelectorAll && node.querySelectorAll('img[data-id]');
if (nestedImages && nestedImages.length > 0) {
nestedImages.forEach((img) => {
const id = img.dataset.id;
if (!imageMap[id]) {
imageMap[id] = img;
console.log('New nested image added:', id);
}
});
}
}
}
});
}
}
});
// Start observing the document body for added nodes
observer.observe(document.body, { childList: true, subtree: true });
(function() {
// Configuration
const imageStatusInterval = 1000; // Interval in milliseconds to check image status
const scrollThreshold = 500; // Distance from bottom of the page to trigger loading
let isFetching = false;
let page = parseInt(document.getElementById('template-data').getAttribute('data-page')) || 1;
let query = document.getElementById('template-data').getAttribute('data-query');
let hardCacheEnabled = document.getElementById('template-data').getAttribute('data-hard-cache-enabled') === 'true';
let imageElements = [];
let imageIds = [];
// Function to check image status
function checkImageStatus() {
const imageIDs = Object.keys(imageMap).filter(id => !loadedImageIDs.has(id));
if (imageIDs.length === 0) {
console.log('All images loaded.');
if (!hardCacheEnabled) return;
if (imageIds.length === 0) {
// No images to check, do nothing
return;
}
console.log('Checking status for images:', imageIDs); // Debugging
fetch('/image_status?image_ids=' + imageIDs.join(','))
// Send AJAX request to check image status
fetch('/image_status?image_ids=' + imageIds.join(','))
.then(response => response.json())
.then(statusMap => {
console.log('Status map:', statusMap); // Debugging
let imagesStillLoading = false;
for (const [id, url] of Object.entries(statusMap)) {
const img = imageMap[id];
if (url) {
// Append cache-busting query parameter
const cacheBustingUrl = url + '?t=' + new Date().getTime();
if (img.src !== cacheBustingUrl) {
img.src = cacheBustingUrl;
img.onload = function() {
// Image loaded successfully
img.classList.add('loaded');
loadedImageIDs.add(id);
};
img.onerror = function() {
console.error('Failed to load image:', url);
};
}
} else {
imagesStillLoading = true;
imageElements = imageElements.filter(img => {
let id = img.getAttribute('data-id');
if (statusMap[id]) {
// Image is ready, update src
img.src = statusMap[id];
// Remove the image id from the list
imageIds = imageIds.filter(imageId => imageId !== id);
return false; // Remove img from imageElements
}
}
if (imagesStillLoading) {
// Poll again after a delay
setTimeout(checkImageStatus, pollingInterval);
} else {
polling = false;
console.log('All images loaded.');
}
return true; // Keep img in imageElements
});
})
.catch(error => {
console.error('Error checking image status:', error);
// Retry after a delay in case of error
setTimeout(checkImageStatus, pollingInterval * 2);
});
}
// Initialize imageElements and imageIds
if (hardCacheEnabled) {
imageElements = Array.from(document.querySelectorAll('img[data-id]'));
imageIds = imageElements.map(img => img.getAttribute('data-id'));
// Replace images with placeholders
imageElements.forEach(img => {
img.src = '/static/images/placeholder.svg';
});
// Start checking image status
let imageStatusTimer = setInterval(checkImageStatus, imageStatusInterval);
checkImageStatus(); // Initial check
}
// Infinite scrolling
window.addEventListener('scroll', function() {
if (isFetching) return;
if (window.innerHeight + window.scrollY >= document.body.offsetHeight - scrollThreshold) {
// User scrolled near the bottom
isFetching = true;
page += 1;
fetch('/search?q=' + encodeURIComponent(query) + '&t=image&p=' + page + '&ajax=true')
.then(response => response.text())
.then(html => {
// Parse the returned HTML and extract image elements
let parser = new DOMParser();
let doc = parser.parseFromString(html, 'text/html');
let newImages = doc.querySelectorAll('.image');
if (newImages.length > 0) {
let resultsContainer = document.querySelector('.images');
newImages.forEach(imageDiv => {
// Append new images to the container
resultsContainer.appendChild(imageDiv);
// Get the img element
let img = imageDiv.querySelector('img');
if (img) {
if (hardCacheEnabled) {
// Replace image with placeholder
img.src = '/static/images/placeholder.svg';
let id = img.getAttribute('data-id');
imageElements.push(img);
imageIds.push(id);
}
}
});
if (hardCacheEnabled) {
checkImageStatus();
}
isFetching = false;
} else {
// No more images to load
isFetching = false;
// Optionally, remove the scroll event listener if no more pages
// window.removeEventListener('scroll', scrollHandler);
}
})
.catch(error => {
console.error('Error fetching next page:', error);
isFetching = false;
});
}
});
// Remove 'js-enabled' class from content
document.getElementById('content').classList.remove('js-enabled');
})();
</script>
// Start polling every pollingInterval milliseconds
const pollingIntervalId = setInterval(checkImageStatus, pollingInterval);
// Optionally, call checkImageStatus immediately
checkImageStatus();
});
<script type="text/javascript">
// Check if 'js_enabled' is not present in the URL
if (!window.location.search.includes('js_enabled=true')) {
// Redirect to the same URL with 'js_enabled=true'
var separator = window.location.search.length ? '&' : '?';
window.location.href = window.location.href + separator + 'js_enabled=true';
}
</script>
<script>

22
templates/images_only.html Normal file
View file

@ -0,0 +1,22 @@
<!-- Images Grid -->
{{ range $index, $result := .Results }}
<div class="image">
<img
{{if $.HardCacheEnabled}}
src="/static/images/placeholder.svg"
data-id="{{ $result.ID }}"
data-full="{{ $result.ProxyFull }}"
data-proxy-full="{{ $result.ProxyThumb }}"
{{else}}
src="{{ $result.ProxyFull }}"
{{end}}
alt="{{ $result.Title }}"
class="clickable"
/>
<div class="resolution">{{ $result.Width }} × {{ $result.Height }}</div>
<div class="details">
<span class="img_title clickable">{{ $result.Title }}</span>
<a href="{{ $result.Source }}" target="_blank" class="img_source">{{ translate "source" }}</a>
</div>
</div>
{{ end }}