github.com/benoitkugler/goacve@v0.0.0-20201217100549-151ce6e55dc8/server/migration/v2_to_v3/cmd/decompress_docs/main.go (about)

     1  // Script de décompression générale des documents.
     2  // S'applique sur une base v3, et enlève la compression gzip
     3  // des contenus.
     4  package main
     5  
     6  import (
     7  	"bytes"
     8  	"compress/gzip"
     9  	"flag"
    10  	"fmt"
    11  	"io"
    12  	"log"
    13  
    14  	"github.com/benoitkugler/goACVE/server/core/rawdata"
    15  	"github.com/benoitkugler/goACVE/logs"
    16  )
    17  
    18  func main() {
    19  	dev := flag.Bool("dev", true, "which DB to use")
    20  	flag.Parse()
    21  
    22  	logsDB := logs.DBProd
    23  	if *dev {
    24  		logsDB = logs.DBDev
    25  	}
    26  
    27  	fmt.Println("Connecting on", logsDB.Host, logsDB.Name)
    28  
    29  	db, err := rawdata.ConnectDB(logsDB)
    30  	if err != nil {
    31  		log.Fatal(err)
    32  	}
    33  	defer db.Close()
    34  
    35  	tx, err := db.Begin()
    36  	if err != nil {
    37  		log.Fatal(err)
    38  	}
    39  
    40  	var totalLengthCompressed, totalLengthUncompressed int
    41  
    42  	fmt.Println("Downloading documents...")
    43  	rows, err := tx.Query("SELECT id_document FROM contenu_documents")
    44  	if err != nil {
    45  		log.Fatal(err)
    46  	}
    47  	ids, err := rawdata.ScanIds(rows)
    48  	if err != nil {
    49  		log.Fatal(err)
    50  	}
    51  
    52  	for i, idDocument := range ids {
    53  		contenu, _, err := rawdata.SelectContenuDocumentByIdDocument(tx, idDocument)
    54  		if err != nil {
    55  			log.Fatal(err)
    56  		}
    57  		if i%5 == 0 {
    58  			fmt.Printf("\r Decompressing document %d / %d (%d %%) ...", i+1, len(ids), 100*(i+1)/len(ids))
    59  		}
    60  		totalLengthCompressed += len(contenu.Contenu)
    61  		r, err := gzip.NewReader(bytes.NewReader(contenu.Contenu))
    62  		if err != nil {
    63  			log.Fatal(err)
    64  		}
    65  		var uncompressed bytes.Buffer
    66  		_, err = io.Copy(&uncompressed, r)
    67  		if err != nil {
    68  			log.Fatal(err)
    69  		}
    70  		_ = r.Close()
    71  
    72  		totalLengthUncompressed += uncompressed.Len()
    73  		contenu.Contenu = uncompressed.Bytes()
    74  
    75  		_, err = tx.Exec("UPDATE contenu_documents SET contenu = $1 WHERE id_document = $2", contenu.Contenu, contenu.IdDocument)
    76  		if err != nil {
    77  			log.Fatal(err)
    78  		}
    79  
    80  		// les miniatures sont déjà décompressées
    81  	}
    82  
    83  	err = tx.Commit()
    84  	if err != nil {
    85  		log.Fatal(err)
    86  	}
    87  
    88  	fmt.Printf("Compression analysis: %f.03 %% (%d / %d)\n",
    89  		100*float32(totalLengthCompressed)/float32(totalLengthUncompressed), totalLengthCompressed/1000, totalLengthUncompressed/1000)
    90  }