Skip to content

Commit

Permalink
change script
Browse files Browse the repository at this point in the history
  • Loading branch information
@redHJ
redHJ committed May 22, 2018
1 parent 4a97fb7 commit d409119
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 68 deletions.
70 changes: 45 additions & 25 deletions transforms/mutate/script.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,44 +53,69 @@ func (g *Script) Init() error {
func (g *Script) Transform(datas []Data) ([]Data, error) {
var err, ferr error
errCount := 0
keys := GetKeys(g.Key)
// 获取 keys
keysArr := strings.Split(g.Key, ",")
keysDetail := make([][]string, 0)
for _, key := range keysArr {
key = strings.TrimSpace(key)
keys := GetKeys(key)
if len(keys) <= 0 {
continue
}
keysDetail = append(keysDetail, keys)
}
// 获取新建keys
news := GetKeys(g.New)
// 增加error
newsErr := g.New + "_error"
recordErrs := GetKeys(newsErr)

if g.storePath == "" {
g.storePath = g.ScriptPath
}
scriptPath := g.storePath
scriptPath, err := checkPath(g.storePath)
if err != nil {
g.stats.LastError = err.Error()
ferr = fmt.Errorf("find total %v erorrs in transform script, last error info is %v", errCount, err)
return datas, ferr
}

for i := range datas {
var scriptRes string
var gerr error
if scriptPath == "" {
params := []string{scriptPath}
for _, keys := range keysDetail {
val, gerr := GetMapValue(datas[i], keys...)
if gerr != nil {
errCount++
err = fmt.Errorf("transform key %v not exist in data", g.Key)
continue
}
var ok bool
if scriptPath, ok = val.(string); !ok {

valStr, ok := val.(string)
if !ok {
errCount++
err = fmt.Errorf("transform key %v data type is not string", g.Key)
continue
}
g.storePath = scriptPath
params = append(params, valStr)
}

gerr = nil
scriptRes, gerr = getScriptRes(g.Interprepter, g.storePath)
scriptRes, gerr = getScriptRes(g.Interprepter, params)
if gerr != nil {
errCount++
err = gerr
if len(gerr.Error()) > 0 {
// 设置脚本执行结果的错误信息
seterr := SetMapValue(datas[i], gerr.Error(), false, recordErrs...)
if seterr != nil {
errCount++
err = fmt.Errorf("the new key %v already exists ", newsErr)
}
}
continue
}

if len(news) == 0 {
DeleteMapValue(datas[i], keys...)
news = keys
}
// 设置脚本执行结果
seterr := SetMapValue(datas[i], scriptRes, false, news...)
if seterr != nil {
errCount++
Expand All @@ -107,17 +132,12 @@ func (g *Script) Transform(datas []Data) ([]Data, error) {
return datas, ferr
}

func getScriptRes(interpreter string, path string) (string, error) {
path, err := checkPath(path)
if err != nil {
return "", err
}

command := exec.Command(interpreter, path) //初始化Cmd
func getScriptRes(interpreter string, params []string) (string, error) {
command := exec.Command(interpreter, params...) //初始化Cmd

res, err := command.Output()
if err != nil {
return "", fmt.Errorf("%s %s - run script err info is %v", interpreter, path, err)
return "", fmt.Errorf("%s %s - run script err info is: %v", interpreter, params, err)
}

return string(res), nil
Expand Down Expand Up @@ -183,22 +203,22 @@ func (g *Script) SampleConfig() string {
func (g *Script) ConfigOptions() []Option {
return []Option{
transforms.KeyFieldName,
transforms.KeyFieldNew,
transforms.KeyFieldNewRequired,
{
KeyName: "interprepter",
ChooseOnly: false,
Default: "bash",
Required: true,
DefaultNoUse: false,
Description: "脚本执行解释器",
Description: "脚本执行解释器(interprepter)",
Type: transforms.TransformTypeString,
},
{
KeyName: "scriptpath",
ChooseOnly: false,
Default: "",
DefaultNoUse: false,
Description: "指定脚本路径",
Description: "指定脚本路径(scriptpath)",
Type: transforms.TransformTypeString,
},
{
Expand All @@ -207,7 +227,7 @@ func (g *Script) ConfigOptions() []Option {
ChooseOnly: false,
Default: "",
DefaultNoUse: false,
Description: "指定脚本内容",
Description: "指定脚本内容(script)",
Type: transforms.TransformTypeString,
},
}
Expand Down
100 changes: 59 additions & 41 deletions transforms/mutate/script_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,88 +6,106 @@ import (

. "github.com/qiniu/logkit/utils/models"

"strings"

"github.com/stretchr/testify/assert"
)

func TestScriptTransformer(t *testing.T) {
fileName := os.TempDir() + "/scriptFile.sh"
//create file & write file
content := `word=$1
name=$2
if [[ ${word} == "" ]]; then
word="hello"
fi
echo "say: ${word} ${name}"`
createTestFile(fileName, content)
defer os.RemoveAll(fileName)

scriptConf := &Script{
Key: "myscript",
Key: "key1",
New: "myscript",
Interprepter: "bash",
}
scriptConf.Init()
fileName := os.TempDir() + "/scriptFile.sh"
//create file & write file
createTestFile(fileName, "echo \"Hello World!\"")
defer os.RemoveAll(fileName)

data := []Data{{"key1": "value1", "myscript": fileName}, {"key2": "value2", "myscript": fileName}}
scriptConf.ScriptPath = fileName
data := []Data{{"key1": "bye", "other": "other"}, {"key1": "nice", "key2": "qiniu"}}
res, err := scriptConf.Transform(data)
assert.NoError(t, err)
exp := []Data{{"key1": "value1", "myscript": "Hello World!\n"}, {"key2": "value2", "myscript": "Hello World!\n"}}
exp := []Data{{"key1": "bye", "other": "other", "myscript": "say: bye \n"}, {"key1": "nice", "key2": "qiniu", "myscript": "say: nice \n"}}
assert.Equal(t, exp, res)

scriptConf2 := &Script{
Key: "myscript",
Key: "key1, key2",
New: "myscript",
Interprepter: "bash",
}
scriptConf2.Init()
data2 := []Data{{"key1": "value1", "myscript": fileName}}
scriptConf2.ScriptPath = fileName
data2 := []Data{{"key1": "", "key2": "qiniu"}, {"key1": "nice", "key2": "qiniu"}}
res2, err2 := scriptConf2.Transform(data2)
assert.NoError(t, err2)
exp2 := []Data{{"key1": "value1", "myscript": "Hello World!\n"}}
exp2 := []Data{{"key1": "", "key2": "qiniu", "myscript": "say: hello qiniu\n"}, {"key1": "nice", "key2": "qiniu", "myscript": "say: nice qiniu\n"}}
assert.Equal(t, exp2, res2)

scriptConf3 := &Script{
Key: "myscript",
New: "newKey",
Key: "key1..., key2...",
New: "myscript...",
Interprepter: "bash",
}
scriptConf3.Init()
data3 := []Data{{"key1": "value1", "myscript": fileName}, {"key2": "value2", "myscript": fileName}}
scriptConf3.ScriptPath = fileName
data3 := []Data{{"key1": "", "key2": "qiniu"}, {"key1": "nice", "key2": "qiniu"}}
res3, err3 := scriptConf3.Transform(data3)
assert.NoError(t, err3)
exp3 := []Data{{"key1": "value1", "myscript": fileName, "newKey": "Hello World!\n"}, {"key2": "value2", "myscript": fileName, "newKey": "Hello World!\n"}}
exp3 := []Data{{"key1": "", "key2": "qiniu", "myscript": "say: hello qiniu\n"}, {"key1": "nice", "key2": "qiniu", "myscript": "say: nice qiniu\n"}}
assert.Equal(t, exp3, res3)

scriptConf4 := &Script{
Key: "myscript...",
New: "newKey...",
Interprepter: "bash",
}
scriptConf4.Init()
data4 := []Data{{"key1": "value1", "myscript": fileName}}
res4, err4 := scriptConf4.Transform(data4)
assert.NoError(t, err4)
exp4 := []Data{{"key1": "value1", "myscript": fileName, "newKey": "Hello World!\n"}}
assert.Equal(t, exp4, res4)

scriptConf5 := &Script{
Key: "myscript...",
Key: "key1..., ..., ...",
New: "myscript...",
Interprepter: "bash",
ScriptPath: fileName,
}
scriptConf5.Init()
data5 := []Data{{"key1": "value1", "myscript": "fileName"}}
scriptConf5.ScriptPath = fileName
data5 := []Data{{"key1": "", "key2": "qiniu"}, {"key1": "nice", "key2": "qiniu"}}
res5, err5 := scriptConf5.Transform(data5)
assert.NoError(t, err5)
exp5 := []Data{{"key1": "value1", "myscript": "Hello World!\n"}}
exp5 := []Data{{"key1": "", "key2": "qiniu", "myscript": "say: hello \n"}, {"key1": "nice", "key2": "qiniu", "myscript": "say: nice \n"}}
assert.Equal(t, exp5, res5)

scriptConf6 := &Script{
Key: "myscript...",
os.RemoveAll(fileName)
fileName = os.TempDir() + "/scriptFile.sh"
//create file & write file
content2 := `bash /a/b/c.sh`
createTestFile(fileName, content2)

scriptConf4 := &Script{
Key: "key1, key2",
New: "myscript",
Interprepter: "bash",
Script: EncodeString(`echo "hello"`),
}
scriptConf6.Init()
data6 := []Data{{"key1": "value1", "myscript": fileName}}
res6, err6 := scriptConf6.Transform(data6)
assert.NoError(t, err6)
exp6 := []Data{{"key1": "value1", "myscript": "hello\n"}}
assert.Equal(t, exp6, res6)
scriptConf4.Init()
scriptConf4.ScriptPath = fileName
data4 := []Data{{"key1": "", "key2": "qiniu"}, {"key1": "nice", "key2": "qiniu"}}
res4, err4 := scriptConf4.Transform(data4)
assert.NoError(t, err4)
exp4 := []Data{{"key1": "", "key2": "qiniu", "myscript": "say: hello qiniu\n"}, {"key1": "nice", "key2": "qiniu", "myscript": "say: nice qiniu\n"}}
assert.Equal(t, len(exp4), len(res4))
for _, resVal := range res4 {
if val, ok := resVal["myscript_error"]; ok {
if !strings.Contains(val.(string), "run script err info is: exit status 127") {
t.Errorf("expect get myscript_error, but got: %v", resVal)
}
} else {
t.Errorf("expect key: myscript_error, but got: %v", resVal)
}
}

scriptConf7 := &Script{
Key: "myscript...",
New: "res",
Interprepter: "bash",
ScriptPath: fileName,
Script: EncodeString(`echo "hello"`),
Expand All @@ -96,7 +114,7 @@ func TestScriptTransformer(t *testing.T) {
data7 := []Data{{"key1": "value1", "myscript": "fileName"}}
res7, err7 := scriptConf7.Transform(data7)
assert.NoError(t, err7)
exp7 := []Data{{"key1": "value1", "myscript": "hello\n"}}
exp7 := []Data{{"key1": "value1", "myscript": "fileName", "res": "hello\n"}}
assert.Equal(t, exp7, res7)
os.RemoveAll("transformer_scripts")
}
Expand Down
3 changes: 1 addition & 2 deletions transforms/registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ var (
KeyName: "new",
ChooseOnly: false,
Default: "",
Required: false,
Placeholder: "new_field_keyname",
DefaultNoUse: false,
Description: "新的字段名(new)",
Expand All @@ -85,7 +84,7 @@ var (
Required: true,
Placeholder: "new_field_keyname",
DefaultNoUse: false,
Description: "解析后数据的字段名(new)",
Description: "新的字段名(new)",
Type: TransformTypeString,
}
KeyTimezoneoffset = Option{
Expand Down

0 comments on commit d409119

Please sign in to comment.