github.com/chainreactors/fingers@v1.2.1/fingerprinthub/ac_index_test.go (about)

     1  package fingerprinthub
     2  
     3  import (
     4  	"bytes"
     5  	"crypto/tls"
     6  	"fmt"
     7  	"io"
     8  	"net"
     9  	"net/http"
    10  	"sort"
    11  	"strings"
    12  	"sync"
    13  	"testing"
    14  	"time"
    15  
    16  	"github.com/chainreactors/fingers/common"
    17  	"github.com/chainreactors/fingers/resources"
    18  	"github.com/chainreactors/utils/httputils"
    19  )
    20  
    21  func TestTemplateKeywordIndex_Build(t *testing.T) {
    22  	engine, err := NewFingerPrintHubEngine(resources.FingerprinthubWebData, resources.FingerprinthubServiceData)
    23  	if err != nil {
    24  		t.Fatalf("Failed to create engine: %v", err)
    25  	}
    26  
    27  	idx := engine.webTemplateIndex
    28  	if idx == nil {
    29  		t.Fatal("webTemplateIndex is nil")
    30  	}
    31  
    32  	totalTemplates := len(engine.webTemplates)
    33  	fastPathCount := len(idx.fastPath)
    34  
    35  	t.Logf("Total web templates: %d", totalTemplates)
    36  	t.Logf("Fast-path templates (direct AC resolution): %d (%.1f%%)", fastPathCount, float64(fastPathCount)/float64(totalTemplates)*100)
    37  
    38  	if fastPathCount == 0 {
    39  		t.Error("No fast-path templates — unified matching provides no benefit")
    40  	}
    41  }
    42  
    43  func TestTemplateKeywordIndex_Match(t *testing.T) {
    44  	engine, err := NewFingerPrintHubEngine(resources.FingerprinthubWebData, resources.FingerprinthubServiceData)
    45  	if err != nil {
    46  		t.Fatalf("Failed to create engine: %v", err)
    47  	}
    48  
    49  	testBody := "powered by wordpress"
    50  	testHeader := "server: nginx\ncontent-type: text/html\n"
    51  
    52  	mr := engine.webTemplateIndex.Match(testHeader, testBody)
    53  	totalCandidates := len(mr.Matched) + len(mr.NeedsCheck)
    54  	totalTemplates := len(engine.webTemplates)
    55  
    56  	t.Logf("Matched (fast-path): %d", len(mr.Matched))
    57  	t.Logf("NeedsCheck (slow-path): %d", len(mr.NeedsCheck))
    58  	t.Logf("Total candidates: %d / %d (%.1f%% filtered)",
    59  		totalCandidates, totalTemplates,
    60  		float64(totalTemplates-totalCandidates)/float64(totalTemplates)*100)
    61  
    62  	if totalCandidates >= totalTemplates {
    63  		t.Error("AC index did not filter any templates")
    64  	}
    65  }
    66  
    67  // webMatchBaseline runs the original full-iteration WebMatch logic without AC prefiltering.
    68  func (engine *FingerPrintHubEngine) webMatchBaseline(content []byte) common.Frameworks {
    69  	resp := httputils.NewResponseWithRaw(content)
    70  	if resp == nil {
    71  		return make(common.Frameworks)
    72  	}
    73  
    74  	rawBody := httputils.ReadBody(resp)
    75  	rawBodyStr := string(rawBody)
    76  	event := engine.buildInternalEvent(resp, rawBodyStr, len(content))
    77  	frames := make(common.Frameworks)
    78  
    79  	for _, tmpl := range engine.webTemplates {
    80  		requests := tmpl.GetRequests()
    81  		if len(requests) == 0 {
    82  			continue
    83  		}
    84  		for _, req := range requests {
    85  			if req.Matchers == nil || len(req.Matchers) == 0 {
    86  				continue
    87  			}
    88  			matched := engine.matchRequest(req, event)
    89  			if matched {
    90  				name := tmpl.Info.Name
    91  				if name == "" {
    92  					name = tmpl.Id
    93  				}
    94  				frame := common.NewFramework(name, common.FrameFromFingerprintHub)
    95  				if tmpl.Info.Metadata != nil {
    96  					if vendor, ok := tmpl.Info.Metadata["vendor"].(string); ok {
    97  						frame.Attributes.Vendor = vendor
    98  					}
    99  					if product, ok := tmpl.Info.Metadata["product"].(string); ok {
   100  						frame.Attributes.Product = product
   101  					}
   102  				}
   103  				frames.Add(frame)
   104  				break
   105  			}
   106  		}
   107  	}
   108  	return frames
   109  }
   110  
   111  func frameworkNames(f common.Frameworks) []string {
   112  	var names []string
   113  	for name := range f {
   114  		names = append(names, name)
   115  	}
   116  	sort.Strings(names)
   117  	return names
   118  }
   119  
   120  func TestWebMatchConsistency_Static(t *testing.T) {
   121  	engine, err := NewFingerPrintHubEngine(resources.FingerprinthubWebData, resources.FingerprinthubServiceData)
   122  	if err != nil {
   123  		t.Fatalf("Failed to create engine: %v", err)
   124  	}
   125  
   126  	responses := []struct {
   127  		name string
   128  		raw  string
   129  	}{
   130  		{
   131  			"nginx",
   132  			"HTTP/1.1 200 OK\r\nServer: nginx/1.18.0\r\nContent-Type: text/html\r\n\r\n<html><body>Welcome to nginx!</body></html>",
   133  		},
   134  		{
   135  			"apache",
   136  			"HTTP/1.1 200 OK\r\nServer: Apache/2.4.41\r\nContent-Type: text/html\r\n\r\n<html><body>It works!</body></html>",
   137  		},
   138  		{
   139  			"wordpress",
   140  			"HTTP/1.1 200 OK\r\nServer: nginx\r\nContent-Type: text/html\r\n\r\n<html><head><meta name=\"generator\" content=\"WordPress 5.8\" /></head><body>Blog</body></html>",
   141  		},
   142  		{
   143  			"empty",
   144  			"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><body>Hello</body></html>",
   145  		},
   146  	}
   147  
   148  	for _, tc := range responses {
   149  		t.Run(tc.name, func(t *testing.T) {
   150  			content := []byte(tc.raw)
   151  
   152  			baseline := engine.webMatchBaseline(content)
   153  			optimized := engine.WebMatch(content)
   154  
   155  			baseNames := frameworkNames(baseline)
   156  			optNames := frameworkNames(optimized)
   157  
   158  			if len(baseNames) != len(optNames) {
   159  				t.Errorf("Result count mismatch: baseline=%d optimized=%d", len(baseNames), len(optNames))
   160  				t.Logf("Baseline:  %v", baseNames)
   161  				t.Logf("Optimized: %v", optNames)
   162  				return
   163  			}
   164  
   165  			for i := range baseNames {
   166  				if baseNames[i] != optNames[i] {
   167  					t.Errorf("Framework name mismatch at %d: baseline=%q optimized=%q", i, baseNames[i], optNames[i])
   168  				}
   169  			}
   170  
   171  			t.Logf("Matched %d frameworks: %v", len(baseNames), baseNames)
   172  		})
   173  	}
   174  }
   175  
   176  func fetchRawResponse(host string, port int, useTLS bool) ([]byte, error) {
   177  	addr := fmt.Sprintf("%s:%d", host, port)
   178  	scheme := "http"
   179  	if useTLS {
   180  		scheme = "https"
   181  	}
   182  
   183  	client := &http.Client{
   184  		Timeout: 5 * time.Second,
   185  		Transport: &http.Transport{
   186  			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
   187  			DialContext: (&net.Dialer{
   188  				Timeout: 3 * time.Second,
   189  			}).DialContext,
   190  		},
   191  		CheckRedirect: func(req *http.Request, via []*http.Request) error {
   192  			return http.ErrUseLastResponse
   193  		},
   194  	}
   195  
   196  	resp, err := client.Get(fmt.Sprintf("%s://%s/", scheme, addr))
   197  	if err != nil {
   198  		return nil, err
   199  	}
   200  	defer resp.Body.Close()
   201  
   202  	bodyBytes, err := io.ReadAll(resp.Body)
   203  	if err != nil {
   204  		return nil, err
   205  	}
   206  
   207  	var buf bytes.Buffer
   208  	buf.WriteString(fmt.Sprintf("HTTP/%d.%d %d %s\r\n", resp.ProtoMajor, resp.ProtoMinor, resp.StatusCode, resp.Status))
   209  	for key, values := range resp.Header {
   210  		for _, val := range values {
   211  			buf.WriteString(fmt.Sprintf("%s: %s\r\n", key, val))
   212  		}
   213  	}
   214  	buf.WriteString("\r\n")
   215  	buf.Write(bodyBytes)
   216  
   217  	return buf.Bytes(), nil
   218  }
   219  
   220  type collectedResponse struct {
   221  	label   string // "host:port"
   222  	content []byte
   223  }
   224  
   225  // collectResponses fetches HTTP responses from a /24 subnet on given ports.
   226  // Network errors are silently skipped; only successful responses are returned.
   227  func collectResponses(baseIP string, ports []int, concurrency int) []collectedResponse {
   228  	ip := net.ParseIP(baseIP).To4()
   229  	if ip == nil {
   230  		return nil
   231  	}
   232  
   233  	type result struct {
   234  		label   string
   235  		content []byte
   236  	}
   237  	ch := make(chan result, 512)
   238  	sem := make(chan struct{}, concurrency)
   239  	var wg sync.WaitGroup
   240  
   241  	for i := 0; i < 256; i++ {
   242  		targetIP := make(net.IP, 4)
   243  		copy(targetIP, ip)
   244  		targetIP[3] = byte(i)
   245  		host := targetIP.String()
   246  		for _, port := range ports {
   247  			wg.Add(1)
   248  			go func(h string, p int) {
   249  				defer wg.Done()
   250  				sem <- struct{}{}
   251  				defer func() { <-sem }()
   252  				content, err := fetchRawResponse(h, p, p == 443)
   253  				if err != nil {
   254  					return
   255  				}
   256  				ch <- result{fmt.Sprintf("%s:%d", h, p), content}
   257  			}(host, port)
   258  		}
   259  	}
   260  	go func() { wg.Wait(); close(ch) }()
   261  
   262  	var out []collectedResponse
   263  	for r := range ch {
   264  		out = append(out, collectedResponse{r.label, r.content})
   265  	}
   266  	return out
   267  }
   268  
   269  // TestWebMatchConsistency_Live collects real responses then verifies
   270  // baseline and optimized WebMatch produce identical results.
   271  func TestWebMatchConsistency_Live(t *testing.T) {
   272  	if testing.Short() {
   273  		t.Skip("skipping live network test in short mode")
   274  	}
   275  
   276  	t.Log("Phase 1: collecting responses from 101.132.149.35/24 :80,:443 ...")
   277  	responses := collectResponses("101.132.149.35", []int{80, 443}, 50)
   278  	t.Logf("Collected %d responses", len(responses))
   279  	if len(responses) == 0 {
   280  		t.Skip("no live hosts responded")
   281  	}
   282  
   283  	t.Log("Phase 2: matching (baseline vs optimized) ...")
   284  	engine, err := NewFingerPrintHubEngine(resources.FingerprinthubWebData, resources.FingerprinthubServiceData)
   285  	if err != nil {
   286  		t.Fatalf("Failed to create engine: %v", err)
   287  	}
   288  
   289  	var totalMismatch int
   290  	for _, resp := range responses {
   291  		baseline := engine.webMatchBaseline(resp.content)
   292  		optimized := engine.WebMatch(resp.content)
   293  
   294  		baseNames := frameworkNames(baseline)
   295  		optNames := frameworkNames(optimized)
   296  
   297  		if strings.Join(baseNames, ",") != strings.Join(optNames, ",") {
   298  			totalMismatch++
   299  			t.Errorf("[%s] MISMATCH baseline=%v optimized=%v", resp.label, baseNames, optNames)
   300  		} else if len(baseNames) > 0 {
   301  			t.Logf("[%s] OK %v", resp.label, baseNames)
   302  		}
   303  	}
   304  
   305  	t.Logf("Checked: %d, Mismatches: %d", len(responses), totalMismatch)
   306  	if totalMismatch > 0 {
   307  		t.Errorf("%d mismatches found", totalMismatch)
   308  	}
   309  }
   310  
   311  // TestWebMatchBenchmark_Live collects real responses then benchmarks
   312  // baseline vs optimized matching, measuring ONLY match time.
   313  func TestWebMatchBenchmark_Live(t *testing.T) {
   314  	if testing.Short() {
   315  		t.Skip("skipping live benchmark in short mode")
   316  	}
   317  
   318  	t.Log("Phase 1: collecting responses from 101.132.149.35/24 :80,:443 ...")
   319  	responses := collectResponses("101.132.149.35", []int{80, 443}, 50)
   320  	t.Logf("Collected %d responses\n", len(responses))
   321  	if len(responses) == 0 {
   322  		t.Skip("no live hosts responded")
   323  	}
   324  
   325  	engine, err := NewFingerPrintHubEngine(resources.FingerprinthubWebData, resources.FingerprinthubServiceData)
   326  	if err != nil {
   327  		t.Fatalf("Failed to create engine: %v", err)
   328  	}
   329  
   330  	const rounds = 3
   331  
   332  	// Baseline: iterate all 3138 templates per response
   333  	t.Log("Phase 2: baseline matching (full iteration) ...")
   334  	var baselineTotal time.Duration
   335  	var baselineFrameworks int
   336  	for r := 0; r < rounds; r++ {
   337  		start := time.Now()
   338  		for _, resp := range responses {
   339  			frames := engine.webMatchBaseline(resp.content)
   340  			baselineFrameworks += len(frames)
   341  		}
   342  		baselineTotal += time.Since(start)
   343  	}
   344  	baselineAvg := baselineTotal / time.Duration(rounds)
   345  	baselinePerResp := baselineAvg / time.Duration(len(responses))
   346  
   347  	// Optimized: AC unified matching
   348  	t.Log("Phase 3: optimized matching (AC unified) ...")
   349  	var optimizedTotal time.Duration
   350  	var optimizedFrameworks int
   351  	for r := 0; r < rounds; r++ {
   352  		start := time.Now()
   353  		for _, resp := range responses {
   354  			frames := engine.WebMatch(resp.content)
   355  			optimizedFrameworks += len(frames)
   356  		}
   357  		optimizedTotal += time.Since(start)
   358  	}
   359  	optimizedAvg := optimizedTotal / time.Duration(rounds)
   360  	optimizedPerResp := optimizedAvg / time.Duration(len(responses))
   361  
   362  	speedup := float64(baselineAvg) / float64(optimizedAvg)
   363  
   364  	t.Log("")
   365  	t.Log("========== Results ==========")
   366  	t.Logf("Responses:       %d", len(responses))
   367  	t.Logf("Rounds:          %d", rounds)
   368  	t.Log("")
   369  	t.Logf("Baseline total:  %v  (per response: %v)", baselineAvg, baselinePerResp)
   370  	t.Logf("Optimized total: %v  (per response: %v)", optimizedAvg, optimizedPerResp)
   371  	t.Logf("Speedup:         %.1fx", speedup)
   372  	t.Log("")
   373  	t.Logf("Frameworks found (baseline):  %d", baselineFrameworks/rounds)
   374  	t.Logf("Frameworks found (optimized): %d", optimizedFrameworks/rounds)
   375  
   376  	if baselineFrameworks != optimizedFrameworks {
   377  		t.Errorf("Framework count mismatch across %d rounds: baseline=%d optimized=%d",
   378  			rounds, baselineFrameworks, optimizedFrameworks)
   379  	}
   380  }