Skip to content

Commit

Permalink
Make zap files smaller (project-chip#992)
Browse files Browse the repository at this point in the history
* Add fileFormat to the file.
* Add the cli logic.
* Add both builtin metafiles as default and refresh matter test file.
* Wrap up the type 1 format, keep default to 0.
* Add unit test and trigger proper conversion.
* Allow for injected key/value pairs.
* Fix the reportable.
* Fix the reportable change.
* Fix a unit test.
* Add name ordering.
* Upgrade one matter file.
* Convert an all-cluster matter file.
* Convert some more files.
* Add some more unit tests.
  • Loading branch information
tecimovic authored Apr 18, 2023
1 parent f699d85 commit 313e73a
Show file tree
Hide file tree
Showing 22 changed files with 6,583 additions and 27,430 deletions.
2 changes: 1 addition & 1 deletion apack.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"description": "Graphical configuration tool for application and libraries based on Zigbee Cluster Library.",
"path": [".", "node_modules/.bin/", "ZAP.app/Contents/MacOS"],
"requiredFeatureLevel": "apack.core:9",
"featureLevel": 95,
"featureLevel": 96,
"uc.triggerExtension": "zap",
"executable": {
"zap:win32.x86_64": {
Expand Down
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@
"genmeta2": "node src-script/zap-generate.js --genResultFile --stateDirectory ~/.zap/genmeta --appendGenerationSubdirectory -o ./tmp -i ./test/resource/test-meta.zap --packageMatch fuzzy",
"gentest": "node src-script/zap-generate.js --genResultFile --stateDirectory ~/.zap/gentest -z ./zcl-builtin/silabs/zcl.json -g ./test/gen-template/test/gen-test.json -o ./tmp",
"gendotdot": "node src-script/zap-generate.js --genResultFile -z ./zcl-builtin/dotdot/library.xml -g ./test/gen-template/dotdot/dotdot-templates.json -o ./tmp",
"convert": "node src-script/zap-convert.js -o {basename}.zap -z ./zcl-builtin/silabs/zcl.json -g test/gen-template/zigbee/gen-templates.json ./test/resource/isc/*.isc ./test/resource/*.zap",
"convert-dl": "node src-script/zap-convert.js -o {basename}.zap -z ./zcl-builtin/silabs/zcl.json -g test/gen-template/zigbee/gen-templates.json ./test/resource/isc/ha-door-lock.isc",
"convert": "node src-script/zap-convert.js -o {basename}.zap -z ./zcl-builtin/silabs/zcl.json -g test/gen-template/zigbee/gen-templates.json -i ./test/resource/isc/*.isc ./test/resource/*.zap",
"convert-dl": "node src-script/zap-convert.js -o {basename}.zap -z ./zcl-builtin/silabs/zcl.json -g test/gen-template/zigbee/gen-templates.json -i ./test/resource/isc/ha-door-lock.isc",
"refresh-zap": "node src-script/zap-convert.js -z zcl-builtin/silabs/zcl.json test/resource/three-endpoint-device.zap -g test/gen-template/zigbee/gen-templates.json -o test/resource/three-endpoint-device.zap",
"publish-linux": "exec electron-builder -l -p always",
"linuxpack-ui": "./dist/linux-unpacked/zap --zclProperties zcl-builtin/silabs/zcl-test.properties --genTemplateJson test/gen-template/zigbee/gen-templates.json",
Expand Down
2 changes: 1 addition & 1 deletion src-electron/db/query-command.js
Original file line number Diff line number Diff line change
Expand Up @@ -1176,7 +1176,7 @@ ON
WHERE PACKAGE_REF IN (${dbApi.toInClause(packageIds)})
GROUP BY
COMMAND.COMMAND_ID
ORDER BY CODE`
ORDER BY COMMAND.CODE, COMMAND.NAME`
)
.then((rows) => rows.map(dbMapping.map.command))
}
Expand Down
20 changes: 17 additions & 3 deletions src-electron/importexport/export.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ const env = require('../util/env')
const querySession = require('../db/query-session.js')
const queryImpExp = require('../db/query-impexp.js')
const dbEnum = require('../../src-shared/db-enum.js')
const ff = require('./file-format')

async function exportEndpointType(db, endpointType) {
let data = await queryImpExp.exportClustersFromEndpointType(
Expand Down Expand Up @@ -157,12 +158,24 @@ async function exportDataIntoFile(
createBackup: false,
}
) {
let fileFormat = env.defaultFileFormat()

env.logDebug(`Writing state from session ${sessionId} into file ${filePath}`)
let state = await createStateFromDatabase(db, sessionId)
if (options.removeLog) delete state.log

if (fileFormat > 0) {
state.fileFormat = fileFormat
} else {
delete state.fileFormat
}
// avoid unncessary Studio integration id from being saved in file.
state['keyValuePairs'] = state['keyValuePairs'].filter(x => x.key != dbEnum.sessionKey.ideProjectPath)
if (state.keyValuePairs) {
state.keyValuePairs = state.keyValuePairs.filter(
(x) => x.key != dbEnum.sessionKey.ideProjectPath
)
}
state = ff.convertToFile(state)

if (options.removeLog) delete state.log

if (fs.existsSync(filePath)) {
fs.copyFileSync(filePath, filePath + '~')
Expand Down Expand Up @@ -210,6 +223,7 @@ async function getSessionKeyValues(db, sessionId, excludedKeys) {
*/
async function createStateFromDatabase(db, sessionId) {
let state = {
fileFormat: 0,
featureLevel: env.zapVersion().featureLevel,
creator: 'zap',
}
Expand Down
310 changes: 310 additions & 0 deletions src-electron/importexport/file-format.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,310 @@
/**
*
* Copyright (c) 2023 Silicon Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

const _ = require('lodash')
const types = require('../util/types')

// Converts attribute storage string to internal representation
function unpackAttribute(a) {
let data
if (a.includes('=>')) {
data = a.split('=>')[0]
} else {
data = a
}
let toks = data.split(' | ').map((x) => x.trim())
if (toks.length != 12) throw new Error(`Invalid format: ${a}`)

let attr = {}
attr.included = toks[0] === '+' ? 1 : 0
attr.code = types.hexStringToInt(toks[1])
if (toks[2].length == 0) {
attr.mfgCode = null
} else {
attr.mfgCode = types.hexStringToInt(toks[2])
}
attr.side = toks[3]
attr.storageOption = toks[4] === 'Ext' ? 'External' : toks[4]
attr.singleton = toks[5] === 'singleton' ? 1 : 0
attr.bounded = toks[6] === 'bound' ? 1 : 0
attr.defaultValue = toks[7]
attr.reportable = parseInt(toks[8])
attr.minInterval = parseInt(toks[9])
attr.maxInterval = parseInt(toks[10])
attr.reportableChange = parseInt(toks[11])
return attr
}

// Converts attribute object for internal representation.
function packAttribute(a) {
let data = [
a.included === 1 ? '+' : '-',
types.intToHexString(a.code, 2),
a.mfgCode != null ? types.intToHexString(a.mfgCode, 2) : ' ',
a.side,
a.storageOption === 'External' ? 'Ext' : a.storageOption,
a.singleton ? 'singleton' : ' ',
a.bounded ? 'bound' : ' ',
a.defaultValue.padStart(20, ' '),
a.reportable,
a.minInterval,
a.maxInterval,
a.reportableChange,
].join(' | ')
return `${data} => ${a.name} [${a.type}]`
}

// Converts command storage string to internal representation
function unpackCommand(c) {
let data
if (c.includes('=>')) {
data = c.split('=>')[0]
} else {
data = c
}
let toks = data.split(' | ').map((x) => x.trim())
if (toks.length != 5) throw new Error(`Invalid format: ${a}`)

let cmd = {}
cmd.code = types.hexStringToInt(toks[0])
if (toks[1].length == 0) {
cmd.mfgCode = null
} else {
cmd.mfgCode = types.hexStringToInt(toks[1])
}
cmd.source = toks[2]
cmd.incoming = toks[3] === '1' ? 1 : 0
cmd.outgoing = toks[4] === '1' ? 1 : 0
return cmd
}

// Converts command object for file representation.
function packCommand(cmd) {
let data = [
types.intToHexString(cmd.code, 2),
cmd.mfgCode != null ? types.intToHexString(cmd.mfgCode, 2) : ' ',
cmd.source,
cmd.incoming,
cmd.outgoing,
].join(' | ')
return `${data} => ${cmd.name}`
}

// Convert string representation to internal object representation
function unpackEvent(ev) {
let data
if (ev.includes('=>')) {
data = ev.split('=>')[0]
} else {
data = ev
}
let toks = data.split(' | ').map((x) => x.trim())
if (toks.length != 4) throw new Error(`Invalid format: ${a}`)

let evnt = {}
evnt.included = toks[0] === '+' ? 1 : 0
evnt.code = types.hexStringToInt(toks[1])
if (toks[2].length == 0) {
evnt.mfgCode = null
} else {
evnt.mfgCode = types.hexStringToInt(toks[2])
}
evnt.side = toks[3]

return evnt
}

// Converts event object for file representation
function packEvent(ev) {
let data = [
ev.included === 1 ? '+' : '-',
types.intToHexString(ev.code, 2),
ev.mfgCode != null ? types.intToHexString(ev.mfgCode, 2) : ' ',
ev.side,
].join(' | ')
return `${data} => ${ev.name}`
}

// Converts the key value pairs in the file into internal representation
function unpackKeyValuePairs(keyValuePairs) {
let kvps = []
for (let kvp of keyValuePairs) {
if (_.isString(kvp)) {
let pair = kvp.split('=').map((x) => x.trim())
kvps.push({
key: pair[0],
value: pair[1],
})
} else {
kvps.push(kvp)
}
}
return kvps
}

// Packs key value pairs for extenrnal representation
function packKeyValuePairs(keyValuePairs) {
let props = []
for (let kvp of keyValuePairs) {
props.push(`${kvp.key} = ${kvp.value}`)
}
return props
}

// Cleanses toplevel cluster data.
function cleanseCluster(c) {
c.code = '0x' + c.code.toString(16).padStart(4, '0')
if (c.mfgCode != null) {
c.mfgCode = '0x' + c.mfgCode.toString(16).padStart(4, '0')
} else {
delete c.mfgCode
}
}

// Uncleanses the toplevel cluster data.
function uncleanseCluster(c) {
if (_.isString(c.code)) {
let code = c.code
if (code.startsWith('0x')) code = code.substring(2)
c.code = parseInt(code, 16)
}

if (_.isString(c.mfgCode)) {
let code = c.mfgCode
if (code.startsWith('0x')) code = code.substring(2)
c.mfgCode = parseInt(code, 16)
}
}

/**
* This function gets the state from database and converts it for a given file format.
*
* @param {*} state
* @param {*} fileFormat
*/
function convertToFile(state) {
if (state.fileFormat && state.fileFormat > 0) {
// Convert key value pairs
if (state.keyValuePairs) {
state.keyValuePairs = packKeyValuePairs(state.keyValuePairs)
}

for (let ept of state.endpointTypes) {
// Now cleanse the clusters
for (let c of ept.clusters) {
cleanseCluster(c)
}

for (let c of ept.clusters) {
// Now we convert all the attributes...
if (c.attributes) {
let atts = []
for (let a of c.attributes) {
atts.push(packAttribute(a))
}
c.attributes = atts
}

// ... and commands...
if (c.commands) {
let cmds = []
for (let cmd of c.commands) {
cmds.push(packCommand(cmd))
}
c.commands = cmds
}

// ... and events.
if (c.events) {
let evs = []
for (let ev of c.events) {
evs.push(packEvent(ev))
}
c.events = evs
}
}
}

return state
} else {
return state
}
}

/**
* This function gets the JSON from the file, and converts it to the correct database state
*/
function convertFromFile(state) {
if (state.fileFormat && state.fileFormat > 0) {
// Convert key value pairs
if (state.keyValuePairs) {
state.keyValuePairs = unpackKeyValuePairs(state.keyValuePairs)
}

for (let ept of state.endpointTypes) {
// Now uncleanse the clusters
for (let c of ept.clusters) {
uncleanseCluster(c)

// Now we convert all the attributes...
if (c.attributes) {
let atts = []
for (let a of c.attributes) {
if (_.isString(a)) {
atts.push(unpackAttribute(a))
} else {
atts.push(a)
}
}
c.attributes = atts
}

// ... and commands.
if (c.commands) {
let cmds = []
for (let cmd of c.commands) {
if (_.isString(cmd)) {
cmds.push(unpackCommand(cmd))
} else {
cmds.push(cmd)
}
}
c.commands = cmds
}

// ... and events.
if (c.events) {
let evs = []
for (let ev of c.events) {
if (_.isString(ev)) {
evs.push(unpackEvent(ev))
} else {
evs.push(ev)
}
}
c.events = evs
}
}
}
return state
} else {
return state
}
}

exports.convertFromFile = convertFromFile
exports.convertToFile = convertToFile
2 changes: 2 additions & 0 deletions src-electron/importexport/import.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ const querySession = require('../db/query-session.js')
const env = require('../util/env')
const script = require('../util/script')
const dbEnum = require('../../src-shared/db-enum')
const ff = require('./file-format.js')

/**
* Reads the data from the file and resolves with the state object if all is good.
Expand Down Expand Up @@ -88,6 +89,7 @@ async function importDataFromFile(
}
) {
let state = await readDataFromFile(filePath, options.defaultZclMetafile)
state = ff.convertFromFile(state)
try {
await dbApi.dbBeginTransaction(db)
let sid
Expand Down
Loading

0 comments on commit 313e73a

Please sign in to comment.