github.com/rothwerx/packer@v0.9.0/post-processor/compress/post-processor.go (about) 1 package compress 2 3 import ( 4 "archive/tar" 5 "archive/zip" 6 "compress/gzip" 7 "fmt" 8 "io" 9 "os" 10 "path/filepath" 11 "regexp" 12 "runtime" 13 14 "github.com/klauspost/pgzip" 15 "github.com/mitchellh/packer/common" 16 "github.com/mitchellh/packer/helper/config" 17 "github.com/mitchellh/packer/packer" 18 "github.com/mitchellh/packer/template/interpolate" 19 "github.com/pierrec/lz4" 20 ) 21 22 var ( 23 // ErrInvalidCompressionLevel is returned when the compression level passed 24 // to gzip is not in the expected range. See compress/flate for details. 25 ErrInvalidCompressionLevel = fmt.Errorf( 26 "Invalid compression level. Expected an integer from -1 to 9.") 27 28 ErrWrongInputCount = fmt.Errorf( 29 "Can only have 1 input file when not using tar/zip") 30 31 filenamePattern = regexp.MustCompile(`(?:\.([a-z0-9]+))`) 32 ) 33 34 type Config struct { 35 common.PackerConfig `mapstructure:",squash"` 36 37 // Fields from config file 38 OutputPath string `mapstructure:"output"` 39 CompressionLevel int `mapstructure:"compression_level"` 40 KeepInputArtifact bool `mapstructure:"keep_input_artifact"` 41 42 // Derived fields 43 Archive string 44 Algorithm string 45 46 ctx interpolate.Context 47 } 48 49 type PostProcessor struct { 50 config Config 51 } 52 53 func (p *PostProcessor) Configure(raws ...interface{}) error { 54 err := config.Decode(&p.config, &config.DecodeOpts{ 55 Interpolate: true, 56 InterpolateContext: &p.config.ctx, 57 InterpolateFilter: &interpolate.RenderFilter{ 58 Exclude: []string{"output"}, 59 }, 60 }, raws...) 61 if err != nil { 62 return err 63 } 64 65 errs := new(packer.MultiError) 66 67 // If there is no explicit number of Go threads to use, then set it 68 if os.Getenv("GOMAXPROCS") == "" { 69 runtime.GOMAXPROCS(runtime.NumCPU()) 70 } 71 72 if p.config.OutputPath == "" { 73 p.config.OutputPath = "packer_{{.BuildName}}_{{.BuilderType}}" 74 } 75 76 if p.config.CompressionLevel > pgzip.BestCompression { 77 p.config.CompressionLevel = pgzip.BestCompression 78 } 79 // Technically 0 means "don't compress" but I don't know how to 80 // differentiate between "user entered zero" and "user entered nothing". 81 // Also, why bother creating a compressed file with zero compression? 82 if p.config.CompressionLevel == -1 || p.config.CompressionLevel == 0 { 83 p.config.CompressionLevel = pgzip.DefaultCompression 84 } 85 86 if err = interpolate.Validate(p.config.OutputPath, &p.config.ctx); err != nil { 87 errs = packer.MultiErrorAppend( 88 errs, fmt.Errorf("Error parsing target template: %s", err)) 89 } 90 91 p.config.detectFromFilename() 92 93 if len(errs.Errors) > 0 { 94 return errs 95 } 96 97 return nil 98 } 99 100 func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { 101 102 // These are extra variables that will be made available for interpolation. 103 p.config.ctx.Data = map[string]string{ 104 "BuildName": p.config.PackerBuildName, 105 "BuilderType": p.config.PackerBuilderType, 106 } 107 108 target, err := interpolate.Render(p.config.OutputPath, &p.config.ctx) 109 if err != nil { 110 return nil, false, fmt.Errorf("Error interpolating output value: %s", err) 111 } else { 112 fmt.Println(target) 113 } 114 115 keep := p.config.KeepInputArtifact 116 newArtifact := &Artifact{Path: target} 117 118 outputFile, err := os.Create(target) 119 if err != nil { 120 return nil, false, fmt.Errorf( 121 "Unable to create archive %s: %s", target, err) 122 } 123 defer outputFile.Close() 124 125 // Setup output interface. If we're using compression, output is a 126 // compression writer. Otherwise it's just a file. 127 var output io.WriteCloser 128 switch p.config.Algorithm { 129 case "lz4": 130 ui.Say(fmt.Sprintf("Using lz4 compression with %d cores for %s", 131 runtime.GOMAXPROCS(-1), target)) 132 output, err = makeLZ4Writer(outputFile, p.config.CompressionLevel) 133 defer output.Close() 134 case "pgzip": 135 ui.Say(fmt.Sprintf("Using pgzip compression with %d cores for %s", 136 runtime.GOMAXPROCS(-1), target)) 137 output, err = makePgzipWriter(outputFile, p.config.CompressionLevel) 138 defer output.Close() 139 default: 140 output = outputFile 141 } 142 143 compression := p.config.Algorithm 144 if compression == "" { 145 compression = "no compression" 146 } 147 148 // Build an archive, if we're supposed to do that. 149 switch p.config.Archive { 150 case "tar": 151 ui.Say(fmt.Sprintf("Tarring %s with %s", target, compression)) 152 err = createTarArchive(artifact.Files(), output) 153 if err != nil { 154 return nil, keep, fmt.Errorf("Error creating tar: %s", err) 155 } 156 case "zip": 157 ui.Say(fmt.Sprintf("Zipping %s", target)) 158 err = createZipArchive(artifact.Files(), output) 159 if err != nil { 160 return nil, keep, fmt.Errorf("Error creating zip: %s", err) 161 } 162 default: 163 // Filename indicates no tarball (just compress) so we'll do an io.Copy 164 // into our compressor. 165 if len(artifact.Files()) != 1 { 166 return nil, keep, fmt.Errorf( 167 "Can only have 1 input file when not using tar/zip. Found %d "+ 168 "files: %v", len(artifact.Files()), artifact.Files()) 169 } 170 archiveFile := artifact.Files()[0] 171 ui.Say(fmt.Sprintf("Archiving %s with %s", archiveFile, compression)) 172 173 source, err := os.Open(archiveFile) 174 if err != nil { 175 return nil, keep, fmt.Errorf( 176 "Failed to open source file %s for reading: %s", 177 archiveFile, err) 178 } 179 defer source.Close() 180 181 if _, err = io.Copy(output, source); err != nil { 182 return nil, keep, fmt.Errorf("Failed to compress %s: %s", 183 archiveFile, err) 184 } 185 } 186 187 ui.Say(fmt.Sprintf("Archive %s completed", target)) 188 189 return newArtifact, keep, nil 190 } 191 192 func (config *Config) detectFromFilename() { 193 194 extensions := map[string]string{ 195 "tar": "tar", 196 "zip": "zip", 197 "gz": "pgzip", 198 "lz4": "lz4", 199 } 200 201 result := filenamePattern.FindAllStringSubmatch(config.OutputPath, -1) 202 203 // No dots. Bail out with defaults. 204 if len(result) == 0 { 205 config.Algorithm = "pgzip" 206 config.Archive = "tar" 207 return 208 } 209 210 // Parse the last two .groups, if they're there 211 lastItem := result[len(result)-1][1] 212 var nextToLastItem string 213 if len(result) == 1 { 214 nextToLastItem = "" 215 } else { 216 nextToLastItem = result[len(result)-2][1] 217 } 218 219 // Should we make an archive? E.g. tar or zip? 220 if nextToLastItem == "tar" { 221 config.Archive = "tar" 222 } 223 if lastItem == "zip" || lastItem == "tar" { 224 config.Archive = lastItem 225 // Tar or zip is our final artifact. Bail out. 226 return 227 } 228 229 // Should we compress the artifact? 230 algorithm, ok := extensions[lastItem] 231 if ok { 232 config.Algorithm = algorithm 233 // We found our compression algorithm. Bail out. 234 return 235 } 236 237 // We didn't match a known compression format. Default to tar + pgzip 238 config.Algorithm = "pgzip" 239 config.Archive = "tar" 240 return 241 } 242 243 func makeLZ4Writer(output io.WriteCloser, compressionLevel int) (io.WriteCloser, error) { 244 lzwriter := lz4.NewWriter(output) 245 if compressionLevel > gzip.DefaultCompression { 246 lzwriter.Header.HighCompression = true 247 } 248 return lzwriter, nil 249 } 250 251 func makePgzipWriter(output io.WriteCloser, compressionLevel int) (io.WriteCloser, error) { 252 gzipWriter, err := pgzip.NewWriterLevel(output, compressionLevel) 253 if err != nil { 254 return nil, ErrInvalidCompressionLevel 255 } 256 gzipWriter.SetConcurrency(500000, runtime.GOMAXPROCS(-1)) 257 return gzipWriter, nil 258 } 259 260 func createTarArchive(files []string, output io.WriteCloser) error { 261 archive := tar.NewWriter(output) 262 defer archive.Close() 263 264 for _, path := range files { 265 file, err := os.Open(path) 266 if err != nil { 267 return fmt.Errorf("Unable to read file %s: %s", path, err) 268 } 269 defer file.Close() 270 271 fi, err := file.Stat() 272 if err != nil { 273 return fmt.Errorf("Unable to get fileinfo for %s: %s", path, err) 274 } 275 276 header, err := tar.FileInfoHeader(fi, path) 277 if err != nil { 278 return fmt.Errorf("Failed to create tar header for %s: %s", path, err) 279 } 280 281 if err := archive.WriteHeader(header); err != nil { 282 return fmt.Errorf("Failed to write tar header for %s: %s", path, err) 283 } 284 285 if _, err := io.Copy(archive, file); err != nil { 286 return fmt.Errorf("Failed to copy %s data to archive: %s", path, err) 287 } 288 } 289 return nil 290 } 291 292 func createZipArchive(files []string, output io.WriteCloser) error { 293 archive := zip.NewWriter(output) 294 defer archive.Close() 295 296 for _, path := range files { 297 path = filepath.ToSlash(path) 298 299 source, err := os.Open(path) 300 if err != nil { 301 return fmt.Errorf("Unable to read file %s: %s", path, err) 302 } 303 defer source.Close() 304 305 target, err := archive.Create(path) 306 if err != nil { 307 return fmt.Errorf("Failed to add zip header for %s: %s", path, err) 308 } 309 310 _, err = io.Copy(target, source) 311 if err != nil { 312 return fmt.Errorf("Failed to copy %s data to archive: %s", path, err) 313 } 314 } 315 return nil 316 }