Changed self-crawling as experimental, cleanup unused features

node-request-text.go

@@ -1,21 +1,23 @@
+//go:build experimental
+// +build experimental
+
 package main
 
 import (
-	"encoding/json"
+	"bytes"
+	"encoding/binary"
 	"fmt"
 	"time"
 )
 
 var textResultsChan = make(chan []TextSearchResult)
 
-func tryOtherNodesForTextSearch(query, safe, lang string, page int, visitedNodes []string) []TextSearchResult {
-	for _, nodeAddr := range peers {
-		if contains(visitedNodes, nodeAddr) {
-			continue // Skip nodes already visited
-		}
-		results, err := sendTextSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
+// Try other nodes is not defined for every type
+func tryOtherNodesForTextSearch(query, safe, lang string, page int) []TextSearchResult {
+	for _, nodeTarget := range sockets {
+		results, err := sendTextSearchRequestToNode(nodeTarget, query, safe, lang, page)
 		if err != nil {
-			printWarn("Error contacting node %s: %v", nodeAddr, err)
+			printWarn("Error contacting node %s: %v", nodeTarget, err)
 			continue
 		}
 		if len(results) > 0 {
@@ -25,60 +27,102 @@ func tryOtherNodesForTextSearch(query, safe, lang string, page int, visitedNodes
 	return nil
 }
 
-func sendTextSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]TextSearchResult, error) {
-	visitedNodes = append(visitedNodes, nodeAddr)
-	searchParams := struct {
-		Query        string   `json:"query"`
-		Safe         string   `json:"safe"`
-		Lang         string   `json:"lang"`
-		Page         int      `json:"page"`
-		ResponseAddr string   `json:"responseAddr"`
-		VisitedNodes []string `json:"visitedNodes"`
-	}{
-		Query:        query,
-		Safe:         safe,
-		Lang:         lang,
-		Page:         page,
-		ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
-		VisitedNodes: visitedNodes,
-	}
-
-	msgBytes, err := json.Marshal(searchParams)
+func sendTextSearchRequestToNode(target, query, safe, lang string, page int) ([]TextSearchResult, error) {
+	payload, err := encodeSearchTextParams(query, safe, lang, page)
 	if err != nil {
-		return nil, fmt.Errorf("failed to marshal search parameters: %v", err)
+		return nil, fmt.Errorf("encode error: %v", err)
 	}
 
 	msg := Message{
-		ID:      hostID,
-		Type:    "search-text",
-		Content: string(msgBytes),
+		ID:      generateMessageID(), // assume function returns uint32
+		Type:    MsgTypeSearchTextRequest,
+		Content: payload,
+		Target:  target,
 	}
 
-	err = sendMessage(nodeAddr, msg)
+	err = sendMessage(msg)
 	if err != nil {
-		return nil, fmt.Errorf("failed to send search request to node %s: %v", nodeAddr, err)
+		return nil, fmt.Errorf("failed to send search request to node %s: %v", target, err)
 	}
 
-	// Wait for results
 	select {
 	case res := <-textResultsChan:
 		return res, nil
 	case <-time.After(20 * time.Second):
-		return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
+		return nil, fmt.Errorf("timeout waiting for results from node %s", target)
 	}
 }
 
 func handleTextResultsMessage(msg Message) {
-	var results []TextSearchResult
-	err := json.Unmarshal([]byte(msg.Content), &results)
+	results, err := decodeTextResults([]byte(msg.Content))
 	if err != nil {
-		printWarn("Error unmarshalling text results: %v", err)
+		printWarn("Error decoding text results: %v", err)
 		return
 	}
-
 	printDebug("Received text results: %+v", results)
-	// Send results to textResultsChan
+
 	go func() {
 		textResultsChan <- results
 	}()
 }
+
+func encodeTextResults(results []TextSearchResult) ([]byte, error) {
+	buf := new(bytes.Buffer)
+
+	if err := binary.Write(buf, binary.BigEndian, uint16(len(results))); err != nil {
+		return nil, err
+	}
+
+	for _, r := range results {
+		if err := writeString(buf, r.URL); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Header); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Description); err != nil {
+			return nil, err
+		}
+		if err := writeString(buf, r.Source); err != nil {
+			return nil, err
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
+func decodeTextResults(data []byte) ([]TextSearchResult, error) {
+	buf := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(buf, binary.BigEndian, &count); err != nil {
+		return nil, err
+	}
+
+	results := make([]TextSearchResult, 0, count)
+	for i := 0; i < int(count); i++ {
+		url, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		header, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		description, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		source, err := readString(buf)
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, TextSearchResult{
+			URL:         url,
+			Header:      header,
+			Description: description,
+			Source:      source,
+		})
+	}
+	return results, nil
+}