Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues 7 Pull requests Projects Releases Packages Wiki Activity Actions

added "VisitedNodes" to message, to prevent re-requesting

Browse source
This commit is contained in:
partisan 2024-08-09 12:59:37 +02:00
parent d34ca730e4
commit 6b99213ec4
7 changed files with 120 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
files.go
View file

@ -82,10 +82,10 @@ func handleFileSearch(w http.ResponseWriter, query, safe, lang string, page int)
Settings: Settings{UxLang: lang, Safe: safe},
}
// Debugging: Print results before rendering template
for _, result := range combinedResults {
fmt.Printf("Title: %s, Magnet: %s\n", result.Title, result.Magnet)
}
// // Debugging: Print results before rendering template
// for _, result := range combinedResults {
// fmt.Printf("Title: %s, Magnet: %s\n", result.Title, result.Magnet)
// }
if err := tmpl.Execute(w, data); err != nil {
log.Printf("Failed to render template: %v", err)
@ -150,15 +150,18 @@ func fetchFileResults(query, safe, lang string, page int) []TorrentResult {
if len(results) == 0 {
log.Printf("No file results found for query: %s, trying other nodes", query)
results = tryOtherNodesForFileSearch(query, safe, lang, page)
results = tryOtherNodesForFileSearch(query, safe, lang, page, []string{hostID})
}
return results
}
func tryOtherNodesForFileSearch(query, safe, lang string, page int) []TorrentResult {
func tryOtherNodesForFileSearch(query, safe, lang string, page int, visitedNodes []string) []TorrentResult {
for _, nodeAddr := range peers {
results, err := sendFileSearchRequestToNode(nodeAddr, query, safe, lang, page)
if contains(visitedNodes, nodeAddr) {
continue // Skip nodes already visited
}
results, err := sendFileSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
continue
@ -170,19 +173,22 @@ func tryOtherNodesForFileSearch(query, safe, lang string, page int) []TorrentRes
return nil
}
func sendFileSearchRequestToNode(nodeAddr, query, safe, lang string, page int) ([]TorrentResult, error) {
func sendFileSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]TorrentResult, error) {
visitedNodes = append(visitedNodes, nodeAddr)
searchParams := struct {
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
VisitedNodes []string `json:"visitedNodes"`
}{
Query: query,
Safe: safe,
Lang: lang,
Page: page,
ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
VisitedNodes: visitedNodes,
}
msgBytes, err := json.Marshal(searchParams)
@ -323,3 +329,12 @@ func sanitizeFileName(name string) string {
sanitized = regexp.MustCompile(`[^a-zA-Z0-9\-\(\)]`).ReplaceAllString(sanitized, "")
return sanitized
}
func contains(slice []string, item string) bool {
for _, v := range slice {
if v == item {
return true
}
}
return false
}

30
forums.go
View file

@ -100,7 +100,7 @@ func PerformRedditSearch(query string, safe string, page int) ([]ForumSearchResu
func handleForumsSearch(w http.ResponseWriter, query, safe, lang string, page int) {
results, err := PerformRedditSearch(query, safe, page)
if err != nil || len(results) == 0 || 0 == 0 { // 0 == 0 to force search by other node
if err != nil || len(results) == 0 { // 0 == 0 to force search by other node
log.Printf("No results from primary search, trying other nodes")
results = tryOtherNodesForForumSearch(query, safe, lang, page)
}
@ -141,7 +141,7 @@ func handleForumsSearch(w http.ResponseWriter, query, safe, lang string, page in
func tryOtherNodesForForumSearch(query, safe, lang string, page int) []ForumSearchResult {
for _, nodeAddr := range peers {
results, err := sendSearchRequestToNode(nodeAddr, query, safe, lang, page)
results, err := sendSearchRequestToNode(nodeAddr, query, safe, lang, page, []string{})
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
continue
@ -153,19 +153,31 @@ func tryOtherNodesForForumSearch(query, safe, lang string, page int) []ForumSear
return nil
}
func sendSearchRequestToNode(nodeAddr, query, safe, lang string, page int) ([]ForumSearchResult, error) {
func sendSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]ForumSearchResult, error) {
// Check if the current node has already been visited
for _, node := range visitedNodes {
if node == hostID {
return nil, fmt.Errorf("loop detected: this node (%s) has already been visited", hostID)
}
}
// Add current node to the list of visited nodes
visitedNodes = append(visitedNodes, hostID)
searchParams := struct {
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
VisitedNodes []string `json:"visitedNodes"`
}{
Query: query,
Safe: safe,
Lang: lang,
Page: page,
ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
VisitedNodes: visitedNodes,
}
msgBytes, err := json.Marshal(searchParams)
@ -188,7 +200,7 @@ func sendSearchRequestToNode(nodeAddr, query, safe, lang string, page int) ([]Fo
select {
case res := <-resultsChan:
return res, nil
case <-time.After(20 * time.Second): // Increased timeout duration
case <-time.After(20 * time.Second):
return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
}
}

42
images.go
View file

@ -10,6 +10,7 @@ import (
)
var imageSearchEngines []SearchEngine
var imageResultsChan = make(chan []ImageSearchResult)
func init() {
imageSearchEngines = []SearchEngine{
@ -124,15 +125,18 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
// If no results found after trying all engines
if len(results) == 0 {
log.Printf("No image results found for query: %s, trying other nodes", query)
results = tryOtherNodesForImageSearch(query, safe, lang, page)
results = tryOtherNodesForImageSearch(query, safe, lang, page, []string{hostID})
}
return results
}
func tryOtherNodesForImageSearch(query, safe, lang string, page int) []ImageSearchResult {
func tryOtherNodesForImageSearch(query, safe, lang string, page int, visitedNodes []string) []ImageSearchResult {
for _, nodeAddr := range peers {
results, err := sendImageSearchRequestToNode(nodeAddr, query, safe, lang, page)
if contains(visitedNodes, nodeAddr) {
continue // Skip nodes already visited
}
results, err := sendImageSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
continue
@ -144,19 +148,22 @@ func tryOtherNodesForImageSearch(query, safe, lang string, page int) []ImageSear
return nil
}
func sendImageSearchRequestToNode(nodeAddr, query, safe, lang string, page int) ([]ImageSearchResult, error) {
func sendImageSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]ImageSearchResult, error) {
visitedNodes = append(visitedNodes, nodeAddr)
searchParams := struct {
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
VisitedNodes []string `json:"visitedNodes"`
}{
Query: query,
Safe: safe,
Lang: lang,
Page: page,
ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
VisitedNodes: visitedNodes,
}
msgBytes, err := json.Marshal(searchParams)
@ -179,7 +186,7 @@ func sendImageSearchRequestToNode(nodeAddr, query, safe, lang string, page int)
select {
case res := <-imageResultsChan:
return res, nil
case <-time.After(30 * time.Second): // Need to handle this better, setting a static number is stupid
case <-time.After(30 * time.Second):
return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
}
}
@ -197,3 +204,18 @@ func wrapImageSearchFunc(f func(string, string, string, int) ([]ImageSearchResul
return searchResults, duration, nil
}
}
func handleImageResultsMessage(msg Message) {
var results []ImageSearchResult
err := json.Unmarshal([]byte(msg.Content), &results)
if err != nil {
log.Printf("Error unmarshalling image results: %v", err)
return
}
log.Printf("Received image results: %+v", results)
// Send results to imageResultsChan
go func() {
imageResultsChan <- results
}()
}

17
node-request-search.go
View file

@ -61,21 +61,4 @@ func fetchForumResults(query, safe, lang string, page int) []ForumSearchResult {
////// IMAGES /////
var imageResultsChan = make(chan []ImageSearchResult)
func handleImageResultsMessage(msg Message) {
var results []ImageSearchResult
err := json.Unmarshal([]byte(msg.Content), &results)
if err != nil {
log.Printf("Error unmarshalling image results: %v", err)
return
}
log.Printf("Received image results: %+v", results)
// Send results to imageResultsChan
go func() {
imageResultsChan <- results
}()
}
////// IMAGES /////

16
node.go
View file

@ -21,9 +21,10 @@ var (
)
type Message struct {
ID string `json:"id"`
Type string `json:"type"`
Content string `json:"content"`
ID string `json:"id"`
Type string `json:"type"`
Content string `json:"content"`
VisitedNodes []string `json:"visitedNodes"`
}
type CrawlerConfig struct {
@ -149,18 +150,17 @@ func interpretMessage(msg Message) {
case "search-file":
handleSearchFileMessage(msg)
case "search-forum":
log.Println("Received search-forum message:", msg.Content)
handleSearchForumMessage(msg)
case "forum-results":
handleForumResultsMessage(msg)
case "text-results":
handleTextResultsMessage(msg) // need to implement
handleTextResultsMessage(msg)
case "image-results":
handleImageResultsMessage(msg) // need to implement
handleImageResultsMessage(msg)
case "video-results":
handleVideoResultsMessage(msg) // need to implement
handleVideoResultsMessage(msg)
case "file-results":
handleFileResultsMessage(msg) // need to implement
handleFileResultsMessage(msg)
default:
fmt.Println("Received unknown message type:", msg.Type)
}

26
text.go
View file

@ -149,7 +149,7 @@ func fetchTextResults(query, safe, lang string, page int) []TextSearchResult {
// If no results found after trying all engines
if len(results) == 0 {
log.Printf("No text results found for query: %s, trying other nodes", query)
results = tryOtherNodesForTextSearch(query, safe, lang, page)
results = tryOtherNodesForTextSearch(query, safe, lang, page, []string{hostID})
}
return results
@ -183,9 +183,12 @@ func wrapTextSearchFunc(f func(string, string, string, int) ([]TextSearchResult,
}
}
func tryOtherNodesForTextSearch(query, safe, lang string, page int) []TextSearchResult {
func tryOtherNodesForTextSearch(query, safe, lang string, page int, visitedNodes []string) []TextSearchResult {
for _, nodeAddr := range peers {
results, err := sendTextSearchRequestToNode(nodeAddr, query, safe, lang, page)
if contains(visitedNodes, nodeAddr) {
continue // Skip nodes already visited
}
results, err := sendTextSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
continue
@ -197,19 +200,22 @@ func tryOtherNodesForTextSearch(query, safe, lang string, page int) []TextSearch
return nil
}
func sendTextSearchRequestToNode(nodeAddr, query, safe, lang string, page int) ([]TextSearchResult, error) {
func sendTextSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]TextSearchResult, error) {
visitedNodes = append(visitedNodes, nodeAddr)
searchParams := struct {
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
VisitedNodes []string `json:"visitedNodes"`
}{
Query: query,
Safe: safe,
Lang: lang,
Page: page,
ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
VisitedNodes: visitedNodes,
}
msgBytes, err := json.Marshal(searchParams)
@ -232,7 +238,7 @@ func sendTextSearchRequestToNode(nodeAddr, query, safe, lang string, page int) (
select {
case res := <-textResultsChan:
return res, nil
case <-time.After(20 * time.Second): // Increased timeout duration
case <-time.After(20 * time.Second):
return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
}
}

24
video.go
View file

@ -155,7 +155,7 @@ func handleVideoSearch(w http.ResponseWriter, query, safe, lang string, page int
results := fetchVideoResults(query, safe, lang, page)
if len(results) == 0 {
log.Printf("No results from primary search, trying other nodes")
results = tryOtherNodesForVideoSearch(query, safe, lang, page)
results = tryOtherNodesForVideoSearch(query, safe, lang, page, []string{hostID})
}
elapsed := time.Since(start)
@ -210,9 +210,12 @@ func fetchVideoResults(query, safe, lang string, page int) []VideoResult {
return results
}
func tryOtherNodesForVideoSearch(query, safe, lang string, page int) []VideoResult {
func tryOtherNodesForVideoSearch(query, safe, lang string, page int, visitedNodes []string) []VideoResult {
for _, nodeAddr := range peers {
results, err := sendVideoSearchRequestToNode(nodeAddr, query, safe, lang, page)
if contains(visitedNodes, nodeAddr) {
continue // Skip nodes already visited
}
results, err := sendVideoSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
continue
@ -224,19 +227,22 @@ func tryOtherNodesForVideoSearch(query, safe, lang string, page int) []VideoResu
return nil
}
func sendVideoSearchRequestToNode(nodeAddr, query, safe, lang string, page int) ([]VideoResult, error) {
func sendVideoSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]VideoResult, error) {
visitedNodes = append(visitedNodes, nodeAddr)
searchParams := struct {
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
Query string `json:"query"`
Safe string `json:"safe"`
Lang string `json:"lang"`
Page int `json:"page"`
ResponseAddr string `json:"responseAddr"`
VisitedNodes []string `json:"visitedNodes"`
}{
Query: query,
Safe: safe,
Lang: lang,
Page: page,
ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
VisitedNodes: visitedNodes,
}
msgBytes, err := json.Marshal(searchParams)