github.com/celestiaorg/celestia-node@v0.15.0-beta.1/share/eds/eds.go (about) 1 package eds 2 3 import ( 4 "bytes" 5 "context" 6 "crypto/sha256" 7 "errors" 8 "fmt" 9 "io" 10 "math" 11 12 "github.com/ipfs/go-cid" 13 "github.com/ipld/go-car" 14 "github.com/ipld/go-car/util" 15 16 "github.com/celestiaorg/celestia-app/pkg/wrapper" 17 "github.com/celestiaorg/nmt" 18 "github.com/celestiaorg/rsmt2d" 19 20 "github.com/celestiaorg/celestia-node/libs/utils" 21 "github.com/celestiaorg/celestia-node/share" 22 "github.com/celestiaorg/celestia-node/share/ipld" 23 ) 24 25 var ErrEmptySquare = errors.New("share: importing empty data") 26 27 // WriteEDS writes the entire EDS into the given io.Writer as CARv1 file. 28 // This includes all shares in quadrant order, followed by all inner nodes of the NMT tree. 29 // Order: [ Carv1Header | Q1 | Q2 | Q3 | Q4 | inner nodes ] 30 // For more information about the header: https://ipld.io/specs/transport/car/carv1/#header 31 func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) (err error) { 32 ctx, span := tracer.Start(ctx, "write-eds") 33 defer func() { 34 utils.SetStatusAndEnd(span, err) 35 }() 36 37 // Creates and writes Carv1Header. Roots are the eds Row + Col roots 38 err = writeHeader(eds, w) 39 if err != nil { 40 return fmt.Errorf("share: writing carv1 header: %w", err) 41 } 42 // Iterates over shares in quadrant order via eds.GetCell 43 err = writeQuadrants(eds, w) 44 if err != nil { 45 return fmt.Errorf("share: writing shares: %w", err) 46 } 47 48 // Iterates over proofs and writes them to the CAR 49 err = writeProofs(ctx, eds, w) 50 if err != nil { 51 return fmt.Errorf("share: writing proofs: %w", err) 52 } 53 return nil 54 } 55 56 // writeHeader creates a CarV1 header using the EDS's Row and Column roots as the list of DAG roots. 57 func writeHeader(eds *rsmt2d.ExtendedDataSquare, w io.Writer) error { 58 rootCids, err := rootsToCids(eds) 59 if err != nil { 60 return fmt.Errorf("getting root cids: %w", err) 61 } 62 63 return car.WriteHeader(&car.CarHeader{ 64 Roots: rootCids, 65 Version: 1, 66 }, w) 67 } 68 69 // writeQuadrants reorders the shares to quadrant order and writes them to the CARv1 file. 70 func writeQuadrants(eds *rsmt2d.ExtendedDataSquare, w io.Writer) error { 71 hasher := nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, ipld.NMTIgnoreMaxNamespace) 72 shares := quadrantOrder(eds) 73 for _, share := range shares { 74 leaf, err := hasher.HashLeaf(share) 75 if err != nil { 76 return fmt.Errorf("hashing share: %w", err) 77 } 78 cid, err := ipld.CidFromNamespacedSha256(leaf) 79 if err != nil { 80 return fmt.Errorf("getting cid from share: %w", err) 81 } 82 err = util.LdWrite(w, cid.Bytes(), share) 83 if err != nil { 84 return fmt.Errorf("writing share to file: %w", err) 85 } 86 } 87 return nil 88 } 89 90 // writeProofs iterates over the in-memory blockstore's keys and writes all inner nodes to the 91 // CARv1 file. 92 func writeProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) error { 93 // check if proofs are collected by ipld.ProofsAdder in previous reconstructions of eds 94 proofs, err := getProofs(ctx, eds) 95 if err != nil { 96 return fmt.Errorf("recomputing proofs: %w", err) 97 } 98 99 for id, proof := range proofs { 100 err := util.LdWrite(w, id.Bytes(), proof) 101 if err != nil { 102 return fmt.Errorf("writing proof to the car: %w", err) 103 } 104 } 105 return nil 106 } 107 108 func getProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare) (map[cid.Cid][]byte, error) { 109 // check if there are proofs collected by ipld.ProofsAdder in previous reconstruction of eds 110 if adder := ipld.ProofsAdderFromCtx(ctx); adder != nil { 111 defer adder.Purge() 112 return adder.Proofs(), nil 113 } 114 115 // recompute proofs from eds 116 shares := eds.Flattened() 117 shareCount := len(shares) 118 if shareCount == 0 { 119 return nil, ErrEmptySquare 120 } 121 odsWidth := int(math.Sqrt(float64(shareCount)) / 2) 122 123 // this adder ignores leaves, so that they are not added to the store we iterate through in 124 // writeProofs 125 adder := ipld.NewProofsAdder(odsWidth * 2) 126 defer adder.Purge() 127 128 eds, err := rsmt2d.ImportExtendedDataSquare( 129 shares, 130 share.DefaultRSMT2DCodec(), 131 wrapper.NewConstructor(uint64(odsWidth), 132 nmt.NodeVisitor(adder.VisitFn())), 133 ) 134 if err != nil { 135 return nil, fmt.Errorf("recomputing data square: %w", err) 136 } 137 // compute roots 138 if _, err = eds.RowRoots(); err != nil { 139 return nil, fmt.Errorf("computing row roots: %w", err) 140 } 141 142 return adder.Proofs(), nil 143 } 144 145 // quadrantOrder reorders the shares in the EDS to quadrant row-by-row order, prepending the 146 // respective namespace to the shares. 147 // e.g. [ Q1 R1 | Q1 R2 | Q1 R3 | Q1 R4 | Q2 R1 | Q2 R2 .... ] 148 func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte { 149 size := eds.Width() * eds.Width() 150 shares := make([][]byte, size) 151 152 quadrantWidth := int(eds.Width() / 2) 153 quadrantSize := quadrantWidth * quadrantWidth 154 for i := 0; i < quadrantWidth; i++ { 155 for j := 0; j < quadrantWidth; j++ { 156 cells := getQuadrantCells(eds, uint(i), uint(j)) 157 innerOffset := i*quadrantWidth + j 158 for quadrant := 0; quadrant < 4; quadrant++ { 159 shares[(quadrant*quadrantSize)+innerOffset] = prependNamespace(quadrant, cells[quadrant]) 160 } 161 } 162 } 163 return shares 164 } 165 166 // getQuadrantCells returns the cell of each EDS quadrant with the passed inner-quadrant coordinates 167 func getQuadrantCells(eds *rsmt2d.ExtendedDataSquare, i, j uint) [][]byte { 168 cells := make([][]byte, 4) 169 quadrantWidth := eds.Width() / 2 170 cells[0] = eds.GetCell(i, j) 171 cells[1] = eds.GetCell(i, j+quadrantWidth) 172 cells[2] = eds.GetCell(i+quadrantWidth, j) 173 cells[3] = eds.GetCell(i+quadrantWidth, j+quadrantWidth) 174 return cells 175 } 176 177 // prependNamespace adds the namespace to the passed share if in the first quadrant, 178 // otherwise it adds the ParitySharesNamespace to the beginning. 179 func prependNamespace(quadrant int, shr share.Share) []byte { 180 namespacedShare := make([]byte, 0, share.NamespaceSize+share.Size) 181 switch quadrant { 182 case 0: 183 return append(append(namespacedShare, share.GetNamespace(shr)...), shr...) 184 case 1, 2, 3: 185 return append(append(namespacedShare, share.ParitySharesNamespace...), shr...) 186 default: 187 panic("invalid quadrant") 188 } 189 } 190 191 // rootsToCids converts the EDS's Row and Column roots to CIDs. 192 func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) { 193 rowRoots, err := eds.RowRoots() 194 if err != nil { 195 return nil, err 196 } 197 colRoots, err := eds.ColRoots() 198 if err != nil { 199 return nil, err 200 } 201 202 roots := make([][]byte, 0, len(rowRoots)+len(colRoots)) 203 roots = append(roots, rowRoots...) 204 roots = append(roots, colRoots...) 205 rootCids := make([]cid.Cid, len(roots)) 206 for i, r := range roots { 207 rootCids[i], err = ipld.CidFromNamespacedSha256(r) 208 if err != nil { 209 return nil, fmt.Errorf("getting cid from root: %w", err) 210 } 211 } 212 return rootCids, nil 213 } 214 215 // ReadEDS reads the first EDS quadrant (1/4) from an io.Reader CAR file. 216 // Only the first quadrant will be read, which represents the original data. 217 // The returned EDS is guaranteed to be full and valid against the DataRoot, otherwise ReadEDS 218 // errors. 219 func ReadEDS(ctx context.Context, r io.Reader, root share.DataHash) (eds *rsmt2d.ExtendedDataSquare, err error) { 220 _, span := tracer.Start(ctx, "read-eds") 221 defer func() { 222 utils.SetStatusAndEnd(span, err) 223 }() 224 225 carReader, err := car.NewCarReader(r) 226 if err != nil { 227 return nil, fmt.Errorf("share: reading car file: %w", err) 228 } 229 230 // car header includes both row and col roots in header 231 odsWidth := len(carReader.Header.Roots) / 4 232 odsSquareSize := odsWidth * odsWidth 233 shares := make([][]byte, odsSquareSize) 234 // the first quadrant is stored directly after the header, 235 // so we can just read the first odsSquareSize blocks 236 for i := 0; i < odsSquareSize; i++ { 237 block, err := carReader.Next() 238 if err != nil { 239 return nil, fmt.Errorf("share: reading next car entry: %w", err) 240 } 241 // the stored first quadrant shares are wrapped with the namespace twice. 242 // we cut it off here, because it is added again while importing to the tree below 243 shares[i] = share.GetData(block.RawData()) 244 } 245 246 // use proofs adder if provided, to cache collected proofs while recomputing the eds 247 var opts []nmt.Option 248 visitor := ipld.ProofsAdderFromCtx(ctx).VisitFn() 249 if visitor != nil { 250 opts = append(opts, nmt.NodeVisitor(visitor)) 251 } 252 253 eds, err = rsmt2d.ComputeExtendedDataSquare( 254 shares, 255 share.DefaultRSMT2DCodec(), 256 wrapper.NewConstructor(uint64(odsWidth), opts...), 257 ) 258 if err != nil { 259 return nil, fmt.Errorf("share: computing eds: %w", err) 260 } 261 262 newDah, err := share.NewRoot(eds) 263 if err != nil { 264 return nil, err 265 } 266 if !bytes.Equal(newDah.Hash(), root) { 267 return nil, fmt.Errorf( 268 "share: content integrity mismatch: imported root %s doesn't match expected root %s", 269 newDah.Hash(), 270 root, 271 ) 272 } 273 return eds, nil 274 }