github.com/qri-io/qri@v0.10.1-0.20220104210721-c771715036cb/base/component/kinds.go (about)

     1  package component
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"encoding/json"
     7  	"fmt"
     8  	"io"
     9  	"io/ioutil"
    10  	"os"
    11  	"path/filepath"
    12  
    13  	"github.com/qri-io/dataset"
    14  	"github.com/qri-io/dataset/detect"
    15  	"github.com/qri-io/dataset/dsio"
    16  	"github.com/qri-io/qfs"
    17  	"github.com/qri-io/qri/base/fill"
    18  	"github.com/qri-io/qri/base/toqtype"
    19  	"gopkg.in/yaml.v2"
    20  )
    21  
    22  // WritePerm is the file permission for the written files
    23  const WritePerm = os.FileMode(int(0644))
    24  
    25  // FilesysComponent represents a collection of components existing as files on a filesystem
    26  type FilesysComponent struct {
    27  	BaseComponent
    28  }
    29  
    30  // Compare compares to another component
    31  func (fc *FilesysComponent) Compare(compare Component) (bool, error) {
    32  	return false, fmt.Errorf("cannot compare filesys component containers")
    33  }
    34  
    35  // IsEmpty returns whether the component collection is empty
    36  func (fc *FilesysComponent) IsEmpty() bool {
    37  	return len(fc.Subcomponents) == 0
    38  }
    39  
    40  // WriteTo writes the component as a file to the directory
    41  func (fc *FilesysComponent) WriteTo(dirPath string) (targetFile string, err error) {
    42  	return "", fmt.Errorf("cannot write filesys component")
    43  }
    44  
    45  // RemoveFrom removes the component file from the directory
    46  func (fc *FilesysComponent) RemoveFrom(dirPath string) error {
    47  	return fmt.Errorf("cannot write filesys component")
    48  }
    49  
    50  // DropDerivedValues drops derived values from the component
    51  func (fc *FilesysComponent) DropDerivedValues() {
    52  	for compName := range fc.BaseComponent.Subcomponents {
    53  		fc.BaseComponent.Subcomponents[compName].DropDerivedValues()
    54  	}
    55  }
    56  
    57  // LoadAndFill loads data from the component source file and assigngs it
    58  func (fc *FilesysComponent) LoadAndFill(ds *dataset.Dataset) error {
    59  	return nil
    60  }
    61  
    62  // StructuredData cannot be returned for a filesystem
    63  func (fc *FilesysComponent) StructuredData() (interface{}, error) {
    64  	return nil, fmt.Errorf("cannot convert filesys to a structured data")
    65  }
    66  
    67  // DatasetComponent represents a dataset with components
    68  type DatasetComponent struct {
    69  	BaseComponent
    70  	Value *dataset.Dataset
    71  }
    72  
    73  // Compare compares to another component
    74  func (dc *DatasetComponent) Compare(compare Component) (bool, error) {
    75  	other, ok := compare.(*DatasetComponent)
    76  	if !ok {
    77  		return false, nil
    78  	}
    79  	if err := dc.LoadAndFill(nil); err != nil {
    80  		return false, err
    81  	}
    82  	if err := compare.LoadAndFill(nil); err != nil {
    83  		return false, err
    84  	}
    85  	return compareComponentData(dc.Value, other.Value)
    86  }
    87  
    88  // WriteTo writes the component as a file to the directory
    89  func (dc *DatasetComponent) WriteTo(dirPath string) (targetFile string, err error) {
    90  	return "", fmt.Errorf("cannot write dataset component")
    91  }
    92  
    93  // RemoveFrom removes the component file from the directory
    94  func (dc *DatasetComponent) RemoveFrom(dirPath string) error {
    95  	return fmt.Errorf("cannot write dataset component")
    96  }
    97  
    98  // DropDerivedValues drops derived values from the component
    99  func (dc *DatasetComponent) DropDerivedValues() {
   100  	for compName := range dc.BaseComponent.Subcomponents {
   101  		if compName == "dataset" {
   102  			continue
   103  		}
   104  		dc.BaseComponent.Subcomponents[compName].DropDerivedValues()
   105  	}
   106  	if dc.Value != nil {
   107  		dc.Value.DropDerivedValues()
   108  	}
   109  }
   110  
   111  // LoadAndFill loads data from the component source file and assigngs it
   112  func (dc *DatasetComponent) LoadAndFill(ds *dataset.Dataset) error {
   113  	return nil
   114  }
   115  
   116  // StructuredData returns the dataset as a map[string]
   117  func (dc *DatasetComponent) StructuredData() (interface{}, error) {
   118  	if err := dc.LoadAndFill(nil); err != nil {
   119  		return nil, err
   120  	}
   121  	return toqtype.StructToMap(dc.Value)
   122  }
   123  
   124  // MetaComponent represents a meta component
   125  type MetaComponent struct {
   126  	BaseComponent
   127  	Value *dataset.Meta
   128  	// Prevent the component from being written. Used for testing.
   129  	DisableSerialization bool
   130  }
   131  
   132  // Compare compares to another component
   133  func (mc *MetaComponent) Compare(compare Component) (bool, error) {
   134  	other, ok := compare.(*MetaComponent)
   135  	if !ok {
   136  		return false, nil
   137  	}
   138  	if err := mc.LoadAndFill(nil); err != nil {
   139  		return false, err
   140  	}
   141  	if err := compare.LoadAndFill(nil); err != nil {
   142  		return false, err
   143  	}
   144  	return compareComponentData(mc.Value, other.Value)
   145  }
   146  
   147  // WriteTo writes the component as a file to the directory
   148  func (mc *MetaComponent) WriteTo(dirPath string) (targetFile string, err error) {
   149  	if mc.DisableSerialization {
   150  		return "", fmt.Errorf("serialization is disabled")
   151  	}
   152  	if err = mc.LoadAndFill(nil); err != nil {
   153  		return
   154  	}
   155  	// Okay to output an empty meta, we do so for `qri init`.
   156  	if mc.Value != nil {
   157  		return writeComponentFile(mc.Value, dirPath, "meta.json")
   158  	}
   159  	return "", nil
   160  }
   161  
   162  // RemoveFrom removes the component file from the directory
   163  func (mc *MetaComponent) RemoveFrom(dirPath string) error {
   164  	// TODO(dlong): Does component have SourceFile set?
   165  	if err := os.Remove(filepath.Join(dirPath, "meta.json")); err != nil && !os.IsNotExist(err) {
   166  		return err
   167  	}
   168  	return nil
   169  }
   170  
   171  // DropDerivedValues drops derived values from the component
   172  func (mc *MetaComponent) DropDerivedValues() {
   173  	mc.Value.DropDerivedValues()
   174  }
   175  
   176  // LoadAndFill loads data from the component source file and assigngs it
   177  func (mc *MetaComponent) LoadAndFill(ds *dataset.Dataset) error {
   178  	if mc.Base().IsLoaded {
   179  		return nil
   180  	}
   181  	if mc.Value != nil {
   182  		mc.Base().IsLoaded = true
   183  		return nil
   184  	}
   185  	fields, err := mc.Base().LoadFile()
   186  	if err != nil {
   187  		return err
   188  	}
   189  	mc.Value = &dataset.Meta{}
   190  	if err := fill.Struct(fields, mc.Value); err != nil {
   191  		return err
   192  	}
   193  	if ds != nil {
   194  		ds.Meta = mc.Value
   195  	}
   196  	return nil
   197  }
   198  
   199  // StructuredData returns the meta as a map[string]
   200  func (mc *MetaComponent) StructuredData() (interface{}, error) {
   201  	if err := mc.LoadAndFill(nil); err != nil {
   202  		return nil, err
   203  	}
   204  	return toqtype.StructToMap(mc.Value)
   205  }
   206  
   207  // StructureComponent represents a structure component
   208  type StructureComponent struct {
   209  	BaseComponent
   210  	Value           *dataset.Structure
   211  	SchemaInference func(*dataset.Dataset) (map[string]interface{}, error)
   212  }
   213  
   214  // Compare compares to another component
   215  func (sc *StructureComponent) Compare(compare Component) (bool, error) {
   216  	other, ok := compare.(*StructureComponent)
   217  	if !ok {
   218  		return false, nil
   219  	}
   220  	if err := sc.LoadAndFill(nil); err != nil {
   221  		return false, err
   222  	}
   223  	if err := compare.LoadAndFill(nil); err != nil {
   224  		return false, err
   225  	}
   226  	// TODO(dlong): DropDerivedValues should not be used here, but lazy evaluation requires it.
   227  	sc.Value.DropDerivedValues()
   228  	other.Value.DropDerivedValues()
   229  	return compareComponentData(sc.Value, other.Value)
   230  }
   231  
   232  // WriteTo writes the component as a file to the directory
   233  func (sc *StructureComponent) WriteTo(dirPath string) (targetFile string, err error) {
   234  	if err = sc.LoadAndFill(nil); err != nil {
   235  		return
   236  	}
   237  	if sc.Value != nil && !sc.Value.IsEmpty() {
   238  		return writeComponentFile(sc.Value, dirPath, "structure.json")
   239  	}
   240  	return "", nil
   241  }
   242  
   243  // RemoveFrom removes the component file from the directory
   244  func (sc *StructureComponent) RemoveFrom(dirPath string) error {
   245  	if err := os.Remove(filepath.Join(dirPath, "structure.json")); err != nil && !os.IsNotExist(err) {
   246  		return err
   247  	}
   248  	return nil
   249  }
   250  
   251  // DropDerivedValues drops derived values from the component
   252  func (sc *StructureComponent) DropDerivedValues() {
   253  	sc.Value.DropDerivedValues()
   254  }
   255  
   256  // LoadAndFill loads data from the component source file and assigngs it
   257  func (sc *StructureComponent) LoadAndFill(ds *dataset.Dataset) error {
   258  	if sc.Base().IsLoaded {
   259  		return nil
   260  	}
   261  	if sc.Value != nil {
   262  		sc.Base().IsLoaded = true
   263  		return nil
   264  	}
   265  	fields, err := sc.Base().LoadFile()
   266  	if err != nil {
   267  		return err
   268  	}
   269  	sc.Value = &dataset.Structure{}
   270  	if err := fill.Struct(fields, sc.Value); err != nil {
   271  		return err
   272  	}
   273  	if ds != nil {
   274  		ds.Structure = sc.Value
   275  	}
   276  	return nil
   277  }
   278  
   279  // StructuredData returns the structure as a map[string]
   280  func (sc *StructureComponent) StructuredData() (interface{}, error) {
   281  	if err := sc.LoadAndFill(nil); err != nil {
   282  		return nil, err
   283  	}
   284  	return toqtype.StructToMap(sc.Value)
   285  }
   286  
   287  // CommitComponent represents a commit component
   288  type CommitComponent struct {
   289  	BaseComponent
   290  	Value *dataset.Commit
   291  }
   292  
   293  // Compare compares to another component
   294  func (cc *CommitComponent) Compare(compare Component) (bool, error) {
   295  	other, ok := compare.(*CommitComponent)
   296  	if !ok {
   297  		return false, nil
   298  	}
   299  	if err := cc.LoadAndFill(nil); err != nil {
   300  		return false, err
   301  	}
   302  	if err := compare.LoadAndFill(nil); err != nil {
   303  		return false, err
   304  	}
   305  	return compareComponentData(cc.Value, other.Value)
   306  }
   307  
   308  // WriteTo writes the component as a file to the directory
   309  func (cc *CommitComponent) WriteTo(dirPath string) (targetFile string, err error) {
   310  	if err = cc.LoadAndFill(nil); err != nil {
   311  		return
   312  	}
   313  	if cc.Value != nil && !cc.Value.IsEmpty() {
   314  		return writeComponentFile(cc.Value, dirPath, "commit.json")
   315  	}
   316  	return "", nil
   317  }
   318  
   319  // RemoveFrom removes the component file from the directory
   320  func (cc *CommitComponent) RemoveFrom(dirPath string) error {
   321  	if err := os.Remove(filepath.Join(dirPath, "commit.json")); err != nil && !os.IsNotExist(err) {
   322  		return err
   323  	}
   324  	return nil
   325  }
   326  
   327  // DropDerivedValues drops derived values from the component
   328  func (cc *CommitComponent) DropDerivedValues() {
   329  	cc.Value.DropDerivedValues()
   330  }
   331  
   332  // LoadAndFill loads data from the component source file and assigngs it
   333  func (cc *CommitComponent) LoadAndFill(ds *dataset.Dataset) error {
   334  	if cc.Base().IsLoaded {
   335  		return nil
   336  	}
   337  	if cc.Value != nil {
   338  		cc.Base().IsLoaded = true
   339  		return nil
   340  	}
   341  	fields, err := cc.Base().LoadFile()
   342  	if err != nil {
   343  		return err
   344  	}
   345  	cc.Value = &dataset.Commit{}
   346  	if err := fill.Struct(fields, cc.Value); err != nil {
   347  		return err
   348  	}
   349  	if ds != nil {
   350  		ds.Commit = cc.Value
   351  	}
   352  	return nil
   353  }
   354  
   355  // StructuredData returns the commit as a map[string]
   356  func (cc *CommitComponent) StructuredData() (interface{}, error) {
   357  	if err := cc.LoadAndFill(nil); err != nil {
   358  		return nil, err
   359  	}
   360  	return toqtype.StructToMap(cc.Value)
   361  }
   362  
   363  // BodyComponent represents a body component
   364  type BodyComponent struct {
   365  	BaseComponent
   366  	Resolver       qfs.PathResolver
   367  	BodyFile       qfs.File
   368  	Structure      *dataset.Structure
   369  	InferredSchema map[string]interface{}
   370  	Value          interface{}
   371  }
   372  
   373  // NewBodyComponent returns a body component for the given source file
   374  func NewBodyComponent(file string) *BodyComponent {
   375  	return &BodyComponent{
   376  		BaseComponent: BaseComponent{
   377  			SourceFile: file,
   378  			Format:     filepath.Ext(file),
   379  		},
   380  	}
   381  }
   382  
   383  // Compare compares to another component
   384  func (bc *BodyComponent) Compare(compare Component) (bool, error) {
   385  	other, ok := compare.(*BodyComponent)
   386  	if !ok {
   387  		return false, nil
   388  	}
   389  	if err := bc.LoadAndFill(nil); err != nil {
   390  		return false, err
   391  	}
   392  	if err := other.LoadAndFill(nil); err != nil {
   393  		return false, err
   394  	}
   395  	return compareComponentData(bc.Value, other.Value)
   396  }
   397  
   398  // DropDerivedValues drops derived values from the component
   399  func (bc *BodyComponent) DropDerivedValues() {
   400  }
   401  
   402  // LoadAndFill loads data from the component source file and assigngs it
   403  func (bc *BodyComponent) LoadAndFill(ds *dataset.Dataset) error {
   404  	if bc.Value != nil {
   405  		return nil
   406  	}
   407  
   408  	var err error
   409  	var entries dsio.EntryReader
   410  
   411  	// TODO(dlong): Move this condition into a utility function in dataset.dsio.
   412  	// TODO(dlong): Should we pipe ctx into this function, instead of using context.Background?
   413  	if bc.BodyFile != nil {
   414  		bf := bc.BodyFile
   415  		entries, err = dsio.NewEntryReader(bc.Structure, bf)
   416  		if err != nil {
   417  			return err
   418  		}
   419  	} else if bc.Resolver != nil {
   420  		bf, err := bc.Resolver.Get(context.Background(), bc.Base().SourceFile)
   421  		if err != nil {
   422  			return err
   423  		}
   424  		entries, err = dsio.NewEntryReader(bc.Structure, bf)
   425  		if err != nil {
   426  			return err
   427  		}
   428  	} else {
   429  		f, err := os.Open(bc.SourceFile)
   430  		if err != nil {
   431  			return err
   432  		}
   433  		entries, err = OpenEntryReader(f, bc.BaseComponent.Format)
   434  		if err != nil {
   435  			return err
   436  		}
   437  		bc.InferredSchema = entries.Structure().Schema
   438  	}
   439  
   440  	topLevel, err := dsio.GetTopLevelType(entries.Structure())
   441  	if err != nil {
   442  		return err
   443  	}
   444  
   445  	if topLevel == "array" {
   446  		result := make([]interface{}, 0)
   447  		for {
   448  			ent, err := entries.ReadEntry()
   449  			if err != nil {
   450  				if err.Error() == io.EOF.Error() {
   451  					break
   452  				}
   453  				return err
   454  			}
   455  			result = append(result, ent.Value)
   456  		}
   457  		bc.Value = result
   458  	} else {
   459  		result := make(map[string]interface{})
   460  		for {
   461  			ent, err := entries.ReadEntry()
   462  			if err != nil {
   463  				if err.Error() == io.EOF.Error() {
   464  					break
   465  				}
   466  				return err
   467  			}
   468  			result[ent.Key] = ent.Value
   469  		}
   470  		bc.Value = result
   471  	}
   472  
   473  	return nil
   474  }
   475  
   476  // StructuredData returns the body as a map[string] or []interface{}, depending on top-level type
   477  func (bc *BodyComponent) StructuredData() (interface{}, error) {
   478  	if err := bc.LoadAndFill(nil); err != nil {
   479  		return nil, err
   480  	}
   481  	return bc.Value, nil
   482  }
   483  
   484  // WriteTo writes the component as a file to the directory
   485  func (bc *BodyComponent) WriteTo(dirPath string) (targetFile string, err error) {
   486  	if bc.Value == nil {
   487  		err = bc.LoadAndFill(nil)
   488  		if err != nil {
   489  			return
   490  		}
   491  	}
   492  	body := bc.Value
   493  	if bc.Structure == nil {
   494  		return "", fmt.Errorf("cannot write body without a structure")
   495  	}
   496  	data, err := SerializeBody(body, bc.Structure)
   497  	if err != nil {
   498  		return "", err
   499  	}
   500  	bodyFilename := fmt.Sprintf("body.%s", bc.Format)
   501  	targetFile = filepath.Join(dirPath, bodyFilename)
   502  	return targetFile, ioutil.WriteFile(targetFile, data, WritePerm)
   503  }
   504  
   505  // RemoveFrom removes the component file from the directory
   506  func (bc *BodyComponent) RemoveFrom(dirPath string) error {
   507  	bodyFilename := fmt.Sprintf("body.%s", bc.Format)
   508  	if err := os.Remove(filepath.Join(dirPath, bodyFilename)); err != nil && !os.IsNotExist(err) {
   509  		return err
   510  	}
   511  	return nil
   512  }
   513  
   514  // OpenEntryReader opens a entry reader for the file, determining the schema automatically
   515  // TODO(dlong): Move this to dataset.dsio
   516  func OpenEntryReader(file *os.File, format string) (dsio.EntryReader, error) {
   517  	st := dataset.Structure{Format: format}
   518  	schema, _, err := detect.Schema(&st, file)
   519  	if err != nil {
   520  		return nil, err
   521  	}
   522  	file.Seek(0, 0)
   523  	st.Schema = schema
   524  	entries, err := dsio.NewEntryReader(&st, file)
   525  	if err != nil {
   526  		return nil, err
   527  	}
   528  	return entries, nil
   529  }
   530  
   531  // SerializeBody writes the source, which must be an array or object,
   532  // TODO(dlong): Move this to dataset.dsio
   533  func SerializeBody(source interface{}, st *dataset.Structure) ([]byte, error) {
   534  	buff := bytes.Buffer{}
   535  
   536  	// ensure tabular data formats have a schema
   537  	if st.RequiresTabularSchema() && st.Schema == nil {
   538  		schema, err := detect.TabularSchemaFromTabularData(source)
   539  		if err != nil {
   540  			return nil, fmt.Errorf("serializing body: %w", err)
   541  		}
   542  		st2 := &dataset.Structure{
   543  			Schema: schema,
   544  		}
   545  		st2.Assign(st)
   546  		st = st2
   547  	}
   548  
   549  	writer, err := dsio.NewEntryWriter(st, &buff)
   550  	if err != nil {
   551  		return nil, err
   552  	}
   553  	switch data := source.(type) {
   554  	case []interface{}:
   555  		for i, val := range data {
   556  			writer.WriteEntry(dsio.Entry{Index: i, Value: val})
   557  		}
   558  	case map[string]interface{}:
   559  		for key, val := range data {
   560  			writer.WriteEntry(dsio.Entry{Key: key, Value: val})
   561  		}
   562  	}
   563  	writer.Close()
   564  	return buff.Bytes(), nil
   565  }
   566  
   567  // ReadmeComponent represents a readme component
   568  type ReadmeComponent struct {
   569  	BaseComponent
   570  	Resolver qfs.PathResolver
   571  	Value    *dataset.Readme
   572  }
   573  
   574  // Compare compares to another component
   575  func (rc *ReadmeComponent) Compare(compare Component) (bool, error) {
   576  	other, ok := compare.(*ReadmeComponent)
   577  	if !ok {
   578  		return false, nil
   579  	}
   580  	if err := rc.LoadAndFill(nil); err != nil {
   581  		return false, err
   582  	}
   583  	if err := compare.LoadAndFill(nil); err != nil {
   584  		return false, err
   585  	}
   586  	return compareComponentData(rc.Value, other.Value)
   587  }
   588  
   589  // WriteTo writes the component as a file to the directory
   590  func (rc *ReadmeComponent) WriteTo(dirPath string) (targetFile string, err error) {
   591  	if err = rc.LoadAndFill(nil); err != nil {
   592  		return
   593  	}
   594  	if rc.Value != nil && !rc.Value.IsEmpty() {
   595  		targetFile = filepath.Join(dirPath, fmt.Sprintf("readme.%s", rc.Format))
   596  		if err = ioutil.WriteFile(targetFile, []byte(rc.Value.Text), WritePerm); err != nil {
   597  			return
   598  		}
   599  	}
   600  	return "", nil
   601  }
   602  
   603  // RemoveFrom removes the component file from the directory
   604  func (rc *ReadmeComponent) RemoveFrom(dirPath string) error {
   605  	// TODO(dlong): Does component have SourceFile set?
   606  	if err := os.Remove(filepath.Join(dirPath, fmt.Sprintf("readme.%s", rc.Format))); err != nil && !os.IsNotExist(err) {
   607  		return err
   608  	}
   609  	return nil
   610  }
   611  
   612  // DropDerivedValues drops derived values from the component
   613  func (rc *ReadmeComponent) DropDerivedValues() {
   614  	rc.Value.DropDerivedValues()
   615  }
   616  
   617  // LoadAndFill loads data from the component source file and assigns it
   618  func (rc *ReadmeComponent) LoadAndFill(ds *dataset.Dataset) error {
   619  	if rc.Base().IsLoaded {
   620  		return nil
   621  	}
   622  	if rc.Value == nil {
   623  		fields, err := rc.Base().LoadFile()
   624  		if err != nil {
   625  			return err
   626  		}
   627  		rc.Value = &dataset.Readme{}
   628  		if err := fill.Struct(fields, rc.Value); err != nil {
   629  			return err
   630  		}
   631  	}
   632  	rc.Base().IsLoaded = true
   633  
   634  	if rc.Resolver != nil {
   635  		err := rc.Value.InlineScriptFile(context.Background(), rc.Resolver)
   636  		if err != nil {
   637  			return err
   638  		}
   639  	}
   640  
   641  	if ds != nil {
   642  		ds.Readme = rc.Value
   643  	}
   644  	return nil
   645  }
   646  
   647  // StructuredData returns the readme as a map[string]
   648  func (rc *ReadmeComponent) StructuredData() (interface{}, error) {
   649  	if err := rc.LoadAndFill(nil); err != nil {
   650  		return nil, err
   651  	}
   652  	return toqtype.StructToMap(rc.Value)
   653  }
   654  
   655  // TransformComponent represents a transform component
   656  type TransformComponent struct {
   657  	BaseComponent
   658  	Resolver qfs.PathResolver
   659  	Value    *dataset.Transform
   660  }
   661  
   662  // Compare compares to another component
   663  func (tc *TransformComponent) Compare(compare Component) (bool, error) {
   664  	other, ok := compare.(*TransformComponent)
   665  	if !ok {
   666  		return false, nil
   667  	}
   668  	if err := tc.LoadAndFill(nil); err != nil {
   669  		return false, err
   670  	}
   671  	if err := compare.LoadAndFill(nil); err != nil {
   672  		return false, err
   673  	}
   674  
   675  	// TODO (b5) - for now we're only comparing script bytes, which means changes to transform
   676  	// configuration won't be detected by things like status, What's more, because stored transforms include
   677  	// a starlark syntax and a "qri" key, comparing FSI to stored JSON won't be equal
   678  	// Let's clean this up
   679  	return tc.Value.Text == other.Value.Text, nil
   680  }
   681  
   682  // WriteTo writes the component as a file to the directory
   683  func (tc *TransformComponent) WriteTo(dirPath string) (targetFile string, err error) {
   684  	if err = tc.LoadAndFill(nil); err != nil {
   685  		return
   686  	}
   687  	if tc.Value != nil && !tc.Value.IsEmpty() {
   688  		targetFile = filepath.Join(dirPath, fmt.Sprintf("transform.%s", tc.Format))
   689  		if err = ioutil.WriteFile(targetFile, []byte(tc.Value.Text), WritePerm); err != nil {
   690  			return
   691  		}
   692  	}
   693  	return "", nil
   694  }
   695  
   696  // RemoveFrom removes the component file from the directory
   697  func (tc *TransformComponent) RemoveFrom(dirPath string) error {
   698  	// TODO(dlong): Does component have SoutceFile set?
   699  	if err := os.Remove(filepath.Join(dirPath, fmt.Sprintf("transform.%s", tc.Format))); err != nil && !os.IsNotExist(err) {
   700  		return err
   701  	}
   702  	return nil
   703  }
   704  
   705  // DropDerivedValues drops derived values from the component
   706  func (tc *TransformComponent) DropDerivedValues() {
   707  	tc.Value.DropDerivedValues()
   708  }
   709  
   710  // LoadAndFill loads data from the component soutce file and assigns it
   711  func (tc *TransformComponent) LoadAndFill(ds *dataset.Dataset) error {
   712  	if tc.Base().IsLoaded {
   713  		return nil
   714  	}
   715  	if tc.Value == nil {
   716  		fields, err := tc.Base().LoadFile()
   717  		if err != nil {
   718  			return err
   719  		}
   720  		tc.Value = &dataset.Transform{}
   721  		if err := fill.Struct(fields, tc.Value); err != nil {
   722  			return err
   723  		}
   724  	}
   725  	tc.Base().IsLoaded = true
   726  
   727  	if tc.Resolver != nil {
   728  		err := tc.Value.InlineScriptFile(context.Background(), tc.Resolver)
   729  		if err != nil {
   730  			return err
   731  		}
   732  	}
   733  
   734  	if ds != nil {
   735  		ds.Transform = tc.Value
   736  	}
   737  	return nil
   738  }
   739  
   740  // StructuredData returns the transform as a map[string]
   741  func (tc *TransformComponent) StructuredData() (interface{}, error) {
   742  	if err := tc.LoadAndFill(nil); err != nil {
   743  		return nil, err
   744  	}
   745  	return toqtype.StructToMap(tc.Value)
   746  }
   747  
   748  // Base returns the common base data for the component
   749  func (bc *BaseComponent) Base() *BaseComponent {
   750  	return bc
   751  }
   752  
   753  // LoadFile opens the source file for the component and unmarshals it, adds errors for duplicate
   754  // components and parse errors
   755  func (bc *BaseComponent) LoadFile() (map[string]interface{}, error) {
   756  	if bc.IsLoaded {
   757  		return nil, nil
   758  	}
   759  
   760  	data, err := ioutil.ReadFile(bc.SourceFile)
   761  	if err != nil {
   762  		bc.SetErrorAsProblem("file-open", err)
   763  		return nil, err
   764  	}
   765  	bc.IsLoaded = true
   766  	// Parse the file bytes using the specified format
   767  	fields := make(map[string]interface{})
   768  	switch bc.Format {
   769  	case "json":
   770  		if err = json.Unmarshal(data, &fields); err != nil {
   771  			bc.SetErrorAsProblem("parse", err)
   772  			return nil, err
   773  		}
   774  		return fields, nil
   775  	case "yaml":
   776  		if err = yaml.Unmarshal(data, &fields); err != nil {
   777  			bc.SetErrorAsProblem("parse", err)
   778  			return nil, err
   779  		}
   780  		return fields, nil
   781  	case "html", "md", "star":
   782  		fields["text"] = string(data)
   783  		return fields, nil
   784  	}
   785  	return nil, fmt.Errorf("unknown local file format \"%s\"", bc.Format)
   786  }
   787  
   788  // SetErrorAsProblem converts the error into a problem and assigns it
   789  func (bc *BaseComponent) SetErrorAsProblem(kind string, err error) {
   790  	bc.ProblemKind = kind
   791  	bc.ProblemMessage = err.Error()
   792  }
   793  
   794  // GetSubcomponent returns the component with the given name
   795  func (bc *BaseComponent) GetSubcomponent(name string) Component {
   796  	return bc.Subcomponents[name]
   797  }
   798  
   799  // SetSubcomponent constructs a component of the appropriate type and adds it as a subcomponent
   800  func (bc *BaseComponent) SetSubcomponent(name string, base BaseComponent) Component {
   801  	var component Component
   802  	if name == "meta" {
   803  		component = &MetaComponent{BaseComponent: base}
   804  	} else if name == "commit" {
   805  		component = &CommitComponent{BaseComponent: base}
   806  	} else if name == "structure" {
   807  		component = &StructureComponent{BaseComponent: base}
   808  	} else if name == "readme" {
   809  		component = &ReadmeComponent{BaseComponent: base}
   810  	} else if name == "transform" {
   811  		component = &TransformComponent{BaseComponent: base}
   812  	} else if name == "body" {
   813  		component = &BodyComponent{BaseComponent: base}
   814  	} else if name == "dataset" {
   815  		component = &DatasetComponent{BaseComponent: base}
   816  	} else {
   817  		return nil
   818  	}
   819  	if bc.Subcomponents == nil {
   820  		bc.Subcomponents = make(map[string]Component)
   821  	}
   822  	bc.Subcomponents[name] = component
   823  	return component
   824  }
   825  
   826  // RemoveSubcomponent removes the component with the given name
   827  func (bc *BaseComponent) RemoveSubcomponent(name string) {
   828  	delete(bc.Subcomponents, name)
   829  }
   830  
   831  func compareComponentData(first interface{}, second interface{}) (bool, error) {
   832  	left, err := json.Marshal(first)
   833  	if err != nil {
   834  		return false, err
   835  	}
   836  	rite, err := json.Marshal(second)
   837  	if err != nil {
   838  		return false, err
   839  	}
   840  	return string(left) == string(rite), nil
   841  }
   842  
   843  func writeComponentFile(value interface{}, dirPath string, basefile string) (string, error) {
   844  	data, err := json.MarshalIndent(value, "", " ")
   845  	if err != nil {
   846  		return "", err
   847  	}
   848  	// TODO(dlong): How does this relate to Base.SourceFile? Should respect that.
   849  	targetFile := filepath.Join(dirPath, basefile)
   850  	err = ioutil.WriteFile(targetFile, data, WritePerm)
   851  	if err != nil {
   852  		return "", err
   853  	}
   854  	return targetFile, nil
   855  }