github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/pkg/v1workermeta/api.go (about)

     1  // Copyright 2020 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package v1workermeta
    15  
    16  import (
    17  	"os"
    18  	"path/filepath"
    19  	"strconv"
    20  
    21  	"github.com/BurntSushi/toml"
    22  	"github.com/pingcap/tiflow/dm/config"
    23  	"github.com/pingcap/tiflow/dm/pb"
    24  	"github.com/pingcap/tiflow/dm/pkg/terror"
    25  	"github.com/pingcap/tiflow/dm/pkg/utils"
    26  )
    27  
    28  // `var` rather than `const` for testing.
    29  var (
    30  	// metaPath is the meta data path in v1.0.x.
    31  	metaPath = "./dm_worker_meta"
    32  	// dbPath is the levelDB path in v1.0.x.
    33  	dbPath = "./dm_worker_meta/kv"
    34  )
    35  
    36  // v1SubTaskConfig represents the subtask config in v1.0.x.
    37  type v1SubTaskConfig struct {
    38  	config.SubTaskConfig // embed the subtask config in v2.0.x.
    39  
    40  	// NOTE: in v1.0.x, `ChunkFilesize` is `int64`, but in v2.0.x it's `string`.
    41  	// (ref: https://github.com/pingcap/dm/pull/713).
    42  	// if we decode data with v1.0.x from TOML directly,
    43  	// an error of `toml: cannot load TOML value of type int64 into a Go string` will be reported,
    44  	// so we overwrite it with another filed which has the same struct tag `chunk-filesize` here.
    45  	// but if set `chunk-filesize: 64` in a YAML file, both v1.0.x (int64) and v2.0.x (string) can support it.
    46  	ChunkFilesize int64 `yaml:"chunk-filesize" toml:"chunk-filesize" json:"chunk-filesize"` // -F, --chunk-filesize
    47  }
    48  
    49  // GetSubtasksMeta gets all subtasks' meta (config and status) from `dm_worker_meta` in v1.0.x.
    50  func GetSubtasksMeta() (map[string]*pb.V1SubTaskMeta, error) {
    51  	// check if meta data exist.
    52  	if !utils.IsDirExists(metaPath) || !utils.IsDirExists(dbPath) {
    53  		return nil, nil
    54  	}
    55  
    56  	// open levelDB to get subtasks meta.
    57  	db, err := openDB(dbPath, defaultKVConfig)
    58  	if err != nil {
    59  		return nil, err
    60  	}
    61  	defer db.Close()
    62  
    63  	// load subtasks' meta from levelDB.
    64  	meta, err := newMeta(db)
    65  	if err != nil {
    66  		return nil, err
    67  	}
    68  
    69  	return meta.TasksMeta(), nil
    70  }
    71  
    72  // RemoveSubtasksMeta removes subtasks' metadata.
    73  // this is often called after upgraded from v1.0.x to v2.0.x,
    74  // so no need to handle again after re-started the DM-worker process.
    75  func RemoveSubtasksMeta() error {
    76  	// check is a valid v1.0.x meta path.
    77  	if !utils.IsDirExists(metaPath) || !utils.IsDirExists(dbPath) {
    78  		return terror.ErrInvalidV1WorkerMetaPath.Generate(filepath.Abs(metaPath))
    79  	}
    80  
    81  	// try to open levelDB to check.
    82  	db, err := openDB(dbPath, defaultKVConfig)
    83  	if err != nil {
    84  		return terror.ErrInvalidV1WorkerMetaPath.Generate(filepath.Abs(metaPath))
    85  	}
    86  	defer db.Close()
    87  
    88  	return os.RemoveAll(metaPath)
    89  }
    90  
    91  // SubTaskConfigFromV1TOML gets SubTaskConfig from subtask's TOML data with v1.0.x.
    92  func SubTaskConfigFromV1TOML(data []byte) (config.SubTaskConfig, error) {
    93  	var v1Cfg v1SubTaskConfig
    94  	_, err := toml.Decode(string(data), &v1Cfg)
    95  	if err != nil {
    96  		return config.SubTaskConfig{}, terror.ErrConfigTomlTransform.Delegate(err, "decode v1 subtask config from data")
    97  	}
    98  
    99  	cfg := v1Cfg.SubTaskConfig
   100  	// DM v2.0 doesn't support heartbeat, overwrite it to false
   101  	cfg.EnableHeartbeat = false
   102  	cfg.MydumperConfig.ChunkFilesize = strconv.FormatInt(v1Cfg.ChunkFilesize, 10)
   103  	err = cfg.Adjust(true)
   104  	if err != nil {
   105  		return config.SubTaskConfig{}, terror.ErrConfigTomlTransform.Delegate(err, "transform `chunk-filesize`")
   106  	}
   107  
   108  	return cfg, nil
   109  }