github.com/lazyledger/lazyledger-core@v0.35.0-dev.0.20210613111200-4c651f053571/ipfs/plugin/nmt_test.go (about) 1 package plugin 2 3 import ( 4 "bytes" 5 "crypto/sha256" 6 "fmt" 7 "math/rand" 8 "sort" 9 "strings" 10 "testing" 11 12 shell "github.com/ipfs/go-ipfs-api" 13 "github.com/ipfs/go-verifcid" 14 mh "github.com/multiformats/go-multihash" 15 16 "github.com/lazyledger/nmt" 17 "github.com/lazyledger/rsmt2d" 18 ) 19 20 func TestMultihasherIsRegistered(t *testing.T) { 21 if _, ok := mh.Codes[Sha256Namespace8Flagged]; !ok { 22 t.Fatalf("code not registered in multihash.Codes: %X", Sha256Namespace8Flagged) 23 } 24 } 25 26 func TestVerifCidAllowsCustomMultihasher(t *testing.T) { 27 if ok := verifcid.IsGoodHash(Sha256Namespace8Flagged); !ok { 28 t.Fatalf("code not allowed by verifcid verifcid.IsGoodHash(%X): %v", Sha256Namespace8Flagged, ok) 29 } 30 } 31 32 func TestDataSquareRowOrColumnRawInputParserCidEqNmtRoot(t *testing.T) { 33 tests := []struct { 34 name string 35 leafData [][]byte 36 }{ 37 {"16 leaves", generateRandNamespacedRawData(16, namespaceSize, shareSize)}, 38 {"32 leaves", generateRandNamespacedRawData(32, namespaceSize, shareSize)}, 39 {"extended row", generateExtendedRow(t)}, 40 } 41 for _, tt := range tests { 42 t.Run(tt.name, func(t *testing.T) { 43 buf := createByteBufFromRawData(t, tt.leafData) 44 45 gotNodes, err := DataSquareRowOrColumnRawInputParser(buf, 0, 0) 46 if err != nil { 47 t.Errorf("DataSquareRowOrColumnRawInputParser() unexpected error = %v", err) 48 return 49 } 50 51 multiHashOverhead := 4 52 lastNodeCid := gotNodes[len(gotNodes)-1].Cid() 53 gotHash, wantHash := lastNodeCid.Hash(), nmt.Sha256Namespace8FlaggedLeaf(tt.leafData[0]) 54 if !bytes.Equal(gotHash[multiHashOverhead:], wantHash) { 55 t.Errorf("first node's hash does not match the Cid\ngot: %v\nwant: %v", gotHash[multiHashOverhead:], wantHash) 56 } 57 nodePrefixOffset := 1 // leaf / inner node prefix is one byte 58 lastLeafNodeData := gotNodes[len(gotNodes)-1].RawData() 59 if gotData, wantData := lastLeafNodeData[nodePrefixOffset:], tt.leafData[0]; !bytes.Equal(gotData, wantData) { 60 t.Errorf("first node's data does not match the leaf's data\ngot: %v\nwant: %v", gotData, wantData) 61 } 62 }) 63 } 64 } 65 66 func TestNodeCollector(t *testing.T) { 67 tests := []struct { 68 name string 69 leafData [][]byte 70 }{ 71 {"16 leaves", generateRandNamespacedRawData(16, namespaceSize, shareSize)}, 72 {"32 leaves", generateRandNamespacedRawData(32, namespaceSize, shareSize)}, 73 {"extended row", generateExtendedRow(t)}, 74 } 75 for _, tt := range tests { 76 t.Run(tt.name, func(t *testing.T) { 77 collector := newNodeCollector() 78 n := nmt.New(sha256.New, nmt.NamespaceIDSize(namespaceSize), nmt.NodeVisitor(collector.visit)) 79 80 for _, share := range tt.leafData { 81 err := n.Push(share) 82 if err != nil { 83 t.Errorf("nmt.Push() unexpected error = %v", err) 84 return 85 } 86 } 87 88 rootDigest := n.Root() 89 90 gotNodes := collector.ipldNodes() 91 92 rootNodeCid := gotNodes[0].Cid() 93 multiHashOverhead := 4 94 lastNodeHash := rootNodeCid.Hash() 95 if got, want := lastNodeHash[multiHashOverhead:], rootDigest.Bytes(); !bytes.Equal(got, want) { 96 t.Errorf("hashes don't match\ngot: %v\nwant: %v", got, want) 97 } 98 99 if MustCidFromNamespacedSha256(rootDigest.Bytes()).String() != rootNodeCid.String() { 100 t.Error("root cid nod and hash not identical") 101 } 102 103 lastNodeCid := gotNodes[len(gotNodes)-1].Cid() 104 gotHash, wantHash := lastNodeCid.Hash(), nmt.Sha256Namespace8FlaggedLeaf(tt.leafData[0]) 105 if !bytes.Equal(gotHash[multiHashOverhead:], wantHash) { 106 t.Errorf("first node's hash does not match the Cid\ngot: %v\nwant: %v", gotHash[multiHashOverhead:], wantHash) 107 } 108 nodePrefixOffset := 1 // leaf / inner node prefix is one byte 109 lastLeafNodeData := gotNodes[len(gotNodes)-1].RawData() 110 if gotData, wantData := lastLeafNodeData[nodePrefixOffset:], tt.leafData[0]; !bytes.Equal(gotData, wantData) { 111 t.Errorf("first node's data does not match the leaf's data\ngot: %v\nwant: %v", gotData, wantData) 112 } 113 114 // ensure that every leaf was collected 115 hasMap := make(map[string]bool) 116 for _, node := range gotNodes { 117 hasMap[node.Cid().String()] = true 118 } 119 hasher := nmt.NewNmtHasher(sha256.New, namespaceSize, true) 120 for _, leaf := range tt.leafData { 121 leafCid := MustCidFromNamespacedSha256(hasher.HashLeaf(leaf)) 122 _, has := hasMap[leafCid.String()] 123 if !has { 124 t.Errorf("leaf CID not found in collected nodes. missing: %s", leafCid.String()) 125 } 126 } 127 }) 128 } 129 } 130 131 func TestDagPutWithPlugin(t *testing.T) { 132 t.Skip("Requires running ipfs daemon (serving the HTTP Api) with the plugin compiled and installed") 133 134 t.Log("Warning: running this test writes to your local IPFS block store!") 135 136 const numLeaves = 32 137 data := generateRandNamespacedRawData(numLeaves, namespaceSize, shareSize) 138 buf := createByteBufFromRawData(t, data) 139 printFirst := 10 140 t.Logf("first leaf, nid: %x, data: %x...", data[0][:namespaceSize], data[0][namespaceSize:namespaceSize+printFirst]) 141 n := nmt.New(sha256.New) 142 for _, share := range data { 143 err := n.Push(share) 144 if err != nil { 145 t.Errorf("nmt.Push() unexpected error = %v", err) 146 return 147 } 148 } 149 150 sh := shell.NewLocalShell() 151 cid, err := sh.DagPut(buf, "raw", DagParserFormatName) 152 if err != nil { 153 t.Fatalf("DagPut() failed: %v", err) 154 } 155 // convert Nmt tree root to CID and verify it matches the CID returned by DagPut 156 treeRootBytes := n.Root().Bytes() 157 nmtCid, err := CidFromNamespacedSha256(treeRootBytes) 158 if err != nil { 159 t.Fatalf("cidFromNamespacedSha256() failed: %v", err) 160 } 161 if nmtCid.String() != cid { 162 t.Errorf("CIDs from Nmt and plugin do not match: got %v, want: %v", cid, nmtCid.String()) 163 } 164 // print out cid s.t. it can be used on the commandline 165 t.Logf("Stored with cid: %v\n", cid) 166 167 // DagGet leaf by leaf: 168 for i, wantShare := range data { 169 gotLeaf := &nmtLeafNode{} 170 path := leafIdxToPath(cid, i) 171 if err := sh.DagGet(path, gotLeaf); err != nil { 172 t.Errorf("DagGet(%s) failed: %v", path, err) 173 } 174 if gotShare := gotLeaf.Data; !bytes.Equal(gotShare, wantShare) { 175 t.Errorf("DagGet returned different data than pushed, got: %v, want: %v", gotShare, wantShare) 176 } 177 } 178 } 179 180 func generateExtendedRow(t *testing.T) [][]byte { 181 origData := generateRandNamespacedRawData(16, namespaceSize, shareSize) 182 origDataWithoutNamespaces := make([][]byte, 16) 183 for i, share := range origData { 184 origDataWithoutNamespaces[i] = share[namespaceSize:] 185 } 186 187 extendedData, err := rsmt2d.ComputeExtendedDataSquare( 188 origDataWithoutNamespaces, 189 rsmt2d.NewRSGF8Codec(), 190 rsmt2d.NewDefaultTree, 191 ) 192 if err != nil { 193 t.Fatalf("rsmt2d.Encode(): %v", err) 194 return nil 195 } 196 extendedRow := extendedData.Row(0) 197 for i, rowCell := range extendedRow { 198 if i < len(origData)/4 { 199 nid := origData[i][:namespaceSize] 200 extendedRow[i] = append(nid, rowCell...) 201 } else { 202 maxNid := bytes.Repeat([]byte{0xFF}, namespaceSize) 203 extendedRow[i] = append(maxNid, rowCell...) 204 } 205 } 206 return extendedRow 207 } 208 209 //nolint:unused 210 // when it actually used in the code ;) 211 func leafIdxToPath(cid string, idx int) string { 212 // currently this fmt directive assumes 32 leaves: 213 bin := fmt.Sprintf("%05b", idx) 214 path := strings.Join(strings.Split(bin, ""), "/") 215 return cid + "/" + path 216 } 217 218 func createByteBufFromRawData(t *testing.T, leafData [][]byte) *bytes.Buffer { 219 buf := bytes.NewBuffer(make([]byte, 0)) 220 for _, share := range leafData { 221 _, err := buf.Write(share) 222 if err != nil { 223 t.Fatalf("buf.Write() unexpected error = %v", err) 224 return nil 225 } 226 } 227 return buf 228 } 229 230 func generateRandNamespacedRawData(total int, nidSize int, leafSize int) [][]byte { 231 data := make([][]byte, total) 232 for i := 0; i < total; i++ { 233 nid := make([]byte, nidSize) 234 rand.Read(nid) 235 data[i] = nid 236 } 237 sortByteArrays(data) 238 for i := 0; i < total; i++ { 239 d := make([]byte, leafSize) 240 rand.Read(d) 241 data[i] = append(data[i], d...) 242 } 243 244 return data 245 } 246 247 func sortByteArrays(src [][]byte) { 248 sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 }) 249 }