github.com/lazyledger/lazyledger-core@v0.35.0-dev.0.20210613111200-4c651f053571/ipfs/plugin/nmt_test.go (about)

     1  package plugin
     2  
     3  import (
     4  	"bytes"
     5  	"crypto/sha256"
     6  	"fmt"
     7  	"math/rand"
     8  	"sort"
     9  	"strings"
    10  	"testing"
    11  
    12  	shell "github.com/ipfs/go-ipfs-api"
    13  	"github.com/ipfs/go-verifcid"
    14  	mh "github.com/multiformats/go-multihash"
    15  
    16  	"github.com/lazyledger/nmt"
    17  	"github.com/lazyledger/rsmt2d"
    18  )
    19  
    20  func TestMultihasherIsRegistered(t *testing.T) {
    21  	if _, ok := mh.Codes[Sha256Namespace8Flagged]; !ok {
    22  		t.Fatalf("code not registered in multihash.Codes: %X", Sha256Namespace8Flagged)
    23  	}
    24  }
    25  
    26  func TestVerifCidAllowsCustomMultihasher(t *testing.T) {
    27  	if ok := verifcid.IsGoodHash(Sha256Namespace8Flagged); !ok {
    28  		t.Fatalf("code not allowed by verifcid verifcid.IsGoodHash(%X): %v", Sha256Namespace8Flagged, ok)
    29  	}
    30  }
    31  
    32  func TestDataSquareRowOrColumnRawInputParserCidEqNmtRoot(t *testing.T) {
    33  	tests := []struct {
    34  		name     string
    35  		leafData [][]byte
    36  	}{
    37  		{"16 leaves", generateRandNamespacedRawData(16, namespaceSize, shareSize)},
    38  		{"32 leaves", generateRandNamespacedRawData(32, namespaceSize, shareSize)},
    39  		{"extended row", generateExtendedRow(t)},
    40  	}
    41  	for _, tt := range tests {
    42  		t.Run(tt.name, func(t *testing.T) {
    43  			buf := createByteBufFromRawData(t, tt.leafData)
    44  
    45  			gotNodes, err := DataSquareRowOrColumnRawInputParser(buf, 0, 0)
    46  			if err != nil {
    47  				t.Errorf("DataSquareRowOrColumnRawInputParser() unexpected error = %v", err)
    48  				return
    49  			}
    50  
    51  			multiHashOverhead := 4
    52  			lastNodeCid := gotNodes[len(gotNodes)-1].Cid()
    53  			gotHash, wantHash := lastNodeCid.Hash(), nmt.Sha256Namespace8FlaggedLeaf(tt.leafData[0])
    54  			if !bytes.Equal(gotHash[multiHashOverhead:], wantHash) {
    55  				t.Errorf("first node's hash does not match the Cid\ngot: %v\nwant: %v", gotHash[multiHashOverhead:], wantHash)
    56  			}
    57  			nodePrefixOffset := 1 // leaf / inner node prefix is one byte
    58  			lastLeafNodeData := gotNodes[len(gotNodes)-1].RawData()
    59  			if gotData, wantData := lastLeafNodeData[nodePrefixOffset:], tt.leafData[0]; !bytes.Equal(gotData, wantData) {
    60  				t.Errorf("first node's data does not match the leaf's data\ngot: %v\nwant: %v", gotData, wantData)
    61  			}
    62  		})
    63  	}
    64  }
    65  
    66  func TestNodeCollector(t *testing.T) {
    67  	tests := []struct {
    68  		name     string
    69  		leafData [][]byte
    70  	}{
    71  		{"16 leaves", generateRandNamespacedRawData(16, namespaceSize, shareSize)},
    72  		{"32 leaves", generateRandNamespacedRawData(32, namespaceSize, shareSize)},
    73  		{"extended row", generateExtendedRow(t)},
    74  	}
    75  	for _, tt := range tests {
    76  		t.Run(tt.name, func(t *testing.T) {
    77  			collector := newNodeCollector()
    78  			n := nmt.New(sha256.New, nmt.NamespaceIDSize(namespaceSize), nmt.NodeVisitor(collector.visit))
    79  
    80  			for _, share := range tt.leafData {
    81  				err := n.Push(share)
    82  				if err != nil {
    83  					t.Errorf("nmt.Push() unexpected error = %v", err)
    84  					return
    85  				}
    86  			}
    87  
    88  			rootDigest := n.Root()
    89  
    90  			gotNodes := collector.ipldNodes()
    91  
    92  			rootNodeCid := gotNodes[0].Cid()
    93  			multiHashOverhead := 4
    94  			lastNodeHash := rootNodeCid.Hash()
    95  			if got, want := lastNodeHash[multiHashOverhead:], rootDigest.Bytes(); !bytes.Equal(got, want) {
    96  				t.Errorf("hashes don't match\ngot: %v\nwant: %v", got, want)
    97  			}
    98  
    99  			if MustCidFromNamespacedSha256(rootDigest.Bytes()).String() != rootNodeCid.String() {
   100  				t.Error("root cid nod and hash not identical")
   101  			}
   102  
   103  			lastNodeCid := gotNodes[len(gotNodes)-1].Cid()
   104  			gotHash, wantHash := lastNodeCid.Hash(), nmt.Sha256Namespace8FlaggedLeaf(tt.leafData[0])
   105  			if !bytes.Equal(gotHash[multiHashOverhead:], wantHash) {
   106  				t.Errorf("first node's hash does not match the Cid\ngot: %v\nwant: %v", gotHash[multiHashOverhead:], wantHash)
   107  			}
   108  			nodePrefixOffset := 1 // leaf / inner node prefix is one byte
   109  			lastLeafNodeData := gotNodes[len(gotNodes)-1].RawData()
   110  			if gotData, wantData := lastLeafNodeData[nodePrefixOffset:], tt.leafData[0]; !bytes.Equal(gotData, wantData) {
   111  				t.Errorf("first node's data does not match the leaf's data\ngot: %v\nwant: %v", gotData, wantData)
   112  			}
   113  
   114  			// ensure that every leaf was collected
   115  			hasMap := make(map[string]bool)
   116  			for _, node := range gotNodes {
   117  				hasMap[node.Cid().String()] = true
   118  			}
   119  			hasher := nmt.NewNmtHasher(sha256.New, namespaceSize, true)
   120  			for _, leaf := range tt.leafData {
   121  				leafCid := MustCidFromNamespacedSha256(hasher.HashLeaf(leaf))
   122  				_, has := hasMap[leafCid.String()]
   123  				if !has {
   124  					t.Errorf("leaf CID not found in collected nodes. missing: %s", leafCid.String())
   125  				}
   126  			}
   127  		})
   128  	}
   129  }
   130  
   131  func TestDagPutWithPlugin(t *testing.T) {
   132  	t.Skip("Requires running ipfs daemon (serving the HTTP Api) with the plugin compiled and installed")
   133  
   134  	t.Log("Warning: running this test writes to your local IPFS block store!")
   135  
   136  	const numLeaves = 32
   137  	data := generateRandNamespacedRawData(numLeaves, namespaceSize, shareSize)
   138  	buf := createByteBufFromRawData(t, data)
   139  	printFirst := 10
   140  	t.Logf("first leaf, nid: %x, data: %x...", data[0][:namespaceSize], data[0][namespaceSize:namespaceSize+printFirst])
   141  	n := nmt.New(sha256.New)
   142  	for _, share := range data {
   143  		err := n.Push(share)
   144  		if err != nil {
   145  			t.Errorf("nmt.Push() unexpected error = %v", err)
   146  			return
   147  		}
   148  	}
   149  
   150  	sh := shell.NewLocalShell()
   151  	cid, err := sh.DagPut(buf, "raw", DagParserFormatName)
   152  	if err != nil {
   153  		t.Fatalf("DagPut() failed: %v", err)
   154  	}
   155  	// convert Nmt tree root to CID and verify it matches the CID returned by DagPut
   156  	treeRootBytes := n.Root().Bytes()
   157  	nmtCid, err := CidFromNamespacedSha256(treeRootBytes)
   158  	if err != nil {
   159  		t.Fatalf("cidFromNamespacedSha256() failed: %v", err)
   160  	}
   161  	if nmtCid.String() != cid {
   162  		t.Errorf("CIDs from Nmt and plugin do not match: got %v, want: %v", cid, nmtCid.String())
   163  	}
   164  	// print out cid s.t. it can be used on the commandline
   165  	t.Logf("Stored with cid: %v\n", cid)
   166  
   167  	// DagGet leaf by leaf:
   168  	for i, wantShare := range data {
   169  		gotLeaf := &nmtLeafNode{}
   170  		path := leafIdxToPath(cid, i)
   171  		if err := sh.DagGet(path, gotLeaf); err != nil {
   172  			t.Errorf("DagGet(%s) failed: %v", path, err)
   173  		}
   174  		if gotShare := gotLeaf.Data; !bytes.Equal(gotShare, wantShare) {
   175  			t.Errorf("DagGet returned different data than pushed, got: %v, want: %v", gotShare, wantShare)
   176  		}
   177  	}
   178  }
   179  
   180  func generateExtendedRow(t *testing.T) [][]byte {
   181  	origData := generateRandNamespacedRawData(16, namespaceSize, shareSize)
   182  	origDataWithoutNamespaces := make([][]byte, 16)
   183  	for i, share := range origData {
   184  		origDataWithoutNamespaces[i] = share[namespaceSize:]
   185  	}
   186  
   187  	extendedData, err := rsmt2d.ComputeExtendedDataSquare(
   188  		origDataWithoutNamespaces,
   189  		rsmt2d.NewRSGF8Codec(),
   190  		rsmt2d.NewDefaultTree,
   191  	)
   192  	if err != nil {
   193  		t.Fatalf("rsmt2d.Encode(): %v", err)
   194  		return nil
   195  	}
   196  	extendedRow := extendedData.Row(0)
   197  	for i, rowCell := range extendedRow {
   198  		if i < len(origData)/4 {
   199  			nid := origData[i][:namespaceSize]
   200  			extendedRow[i] = append(nid, rowCell...)
   201  		} else {
   202  			maxNid := bytes.Repeat([]byte{0xFF}, namespaceSize)
   203  			extendedRow[i] = append(maxNid, rowCell...)
   204  		}
   205  	}
   206  	return extendedRow
   207  }
   208  
   209  //nolint:unused
   210  // when it actually used in the code ;)
   211  func leafIdxToPath(cid string, idx int) string {
   212  	// currently this fmt directive assumes 32 leaves:
   213  	bin := fmt.Sprintf("%05b", idx)
   214  	path := strings.Join(strings.Split(bin, ""), "/")
   215  	return cid + "/" + path
   216  }
   217  
   218  func createByteBufFromRawData(t *testing.T, leafData [][]byte) *bytes.Buffer {
   219  	buf := bytes.NewBuffer(make([]byte, 0))
   220  	for _, share := range leafData {
   221  		_, err := buf.Write(share)
   222  		if err != nil {
   223  			t.Fatalf("buf.Write() unexpected error = %v", err)
   224  			return nil
   225  		}
   226  	}
   227  	return buf
   228  }
   229  
   230  func generateRandNamespacedRawData(total int, nidSize int, leafSize int) [][]byte {
   231  	data := make([][]byte, total)
   232  	for i := 0; i < total; i++ {
   233  		nid := make([]byte, nidSize)
   234  		rand.Read(nid)
   235  		data[i] = nid
   236  	}
   237  	sortByteArrays(data)
   238  	for i := 0; i < total; i++ {
   239  		d := make([]byte, leafSize)
   240  		rand.Read(d)
   241  		data[i] = append(data[i], d...)
   242  	}
   243  
   244  	return data
   245  }
   246  
   247  func sortByteArrays(src [][]byte) {
   248  	sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 })
   249  }