Skip to content

Commit

Permalink
add pak package
Browse files Browse the repository at this point in the history
  • Loading branch information
kjk committed Oct 8, 2021
1 parent 3fee7c3 commit 11a3ffc
Show file tree
Hide file tree
Showing 8 changed files with 745 additions and 3 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/biuld.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17
go-version: 1.16

- name: Check out source code
uses: actions/checkout@v2
Expand All @@ -18,5 +18,5 @@ jobs:

- name: Staticcheck
run: |
# go get -u honnef.co/go/tools/cmd/staticcheck
# staticcheck ./...
go get -u honnef.co/go/tools/cmd/staticcheck
staticcheck ./...
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ module github.com/kjk/common
go 1.16

require (
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/andybalholm/brotli v1.0.3
github.com/stretchr/testify v1.7.0 // indirect
golang.org/x/mod v0.5.1 // indirect
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect
golang.org/x/tools v0.1.7 // indirect
honnef.co/go/tools v0.2.1 // indirect
)
43 changes: 43 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
Expand All @@ -7,6 +10,46 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.2.1 h1:/EPr//+UMMXwMTkXvCCoaJDq8cpjMO80Ou+L4PDo2mY=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
188 changes: 188 additions & 0 deletions pak/all_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
package pak

import (
"fmt"
"io/ioutil"
"os"
"testing"

"github.com/stretchr/testify/assert"
)

func must(err error) {
if err != nil {
panic(err.Error())
}
}

type test struct {
isFile bool
data []byte
// path is for both data and files
path string
// for files, use to over-ride the path of the file
pathOverride string
size int64
tp string
}

func mkFile(path string) *test {
size, err := getFileSize(path)
must(err)
d, err := ioutil.ReadFile(path)
must(err)
return &test{
isFile: true,
path: path,
size: size,
data: d,
}
}

func mkData(d []byte, path string) *test {
return &test{
data: d,
path: path,
size: int64(len(d)),
}
}

func writeTests(t *testing.T, archivePath string, tests []*test) {
w := NewWriter()

for _, test := range tests {
var meta Metadata
// check additional metadata
if test.tp != "" {
meta.Set("Type", test.tp)
}
if test.isFile {
if test.pathOverride != "" {
meta.Set("Path", test.pathOverride)
}
err := w.AddFile(test.path, meta)
assert.NoError(t, err)
} else {
err := w.AddData(test.data, test.path, meta)
assert.NoError(t, err)
}
}

f, err := os.Create(archivePath)
assert.NoError(t, err)
err = w.Write(f)
assert.NoError(t, err)
err = f.Close()
assert.NoError(t, err)
}

func readAndVerifyTests(t *testing.T, archivePath string, tests []*test) {
a, err := ReadArchive(archivePath)
assert.NoError(t, err)

entries := a.Entries
assert.NoError(t, err)
assert.Equal(t, len(entries), len(tests))

for i, e := range entries {
test := tests[i]
d, err := a.ReadEntry(e)
assert.NoError(t, err)
assert.Equal(t, test.data, d)

path := test.path
if test.pathOverride != "" {
path = test.pathOverride
}

assert.Equal(t, path, e.Path)
v, _ := e.Metadata.Get("Path")
assert.Equal(t, path, v)

assert.Equal(t, test.size, e.Size)
assert.Equal(t, test.data, d)
expMetaSize := 3
if test.tp != "" {
v, _ = e.Metadata.Get("Type")
assert.Equal(t, test.tp, v)
expMetaSize++
}
assert.Equal(t, expMetaSize, e.Metadata.Size())
}
}

func TestWriteRead(t *testing.T) {
tests := []*test{
mkFile("all_test.go"),
mkFile("metadata.go"),
mkFile("reader.go"),
mkFile("writer.go"),
mkData([]byte{}, "empty_file.txt"),
mkFile("readme.md"),
mkData([]byte{0x0, 0x1, 0x3, 0x0}, "some_data.dat"),
}
tests[1].tp = "sum file"
tests[2].pathOverride = "LICENSE.overwritten"

files, err := ioutil.ReadDir(".")
assert.NoError(t, err)
for _, fi := range files {
if !fi.Mode().IsRegular() {
continue
}
d, err := ioutil.ReadFile(fi.Name())
must(err)
test := mkData(d, fi.Name()+".data")
tests = append(tests, test)
}

writeReadArchiveTests(t, "test_archive.txt", tests)
}

func writeReadArchiveTests(t *testing.T, archivePath string, tests []*test) {
writeTests(t, archivePath, tests)

// on error leave the archive file for inspection
defer func() {
if !t.Failed() {
err := os.Remove(archivePath)
if err != nil {
fmt.Printf("os.Remove('%s') failed with '%s'\n", archivePath, err)
}
}
}()
readAndVerifyTests(t, archivePath, tests)

}

func TestBug(t *testing.T) {
tests := []*test{
// siser had issues when reading a record followed by record
// starting with 0xa
mkData([]byte{0xa, 0x1, 0x3, 0x0}, "some_data.dat"),
mkFile("reader.go"),
}

writeReadArchiveTests(t, "test_archive_bug.txt", tests)
}

func TestMetadata(t *testing.T) {
var m Metadata
assert.Equal(t, 0, m.Size())
var added, ok bool
var v string
added = m.Set("Foo", "bar")
assert.True(t, added)
v, ok = m.Get("Foo")
assert.True(t, ok)
assert.Equal(t, v, "bar")
added = m.Set("foo", "bar")
assert.True(t, added) // case insensitive
assert.Equal(t, 2, m.Size())
added = m.Set("Foo", "bar2") // update
assert.False(t, added)
assert.Equal(t, 2, m.Size())
v, ok = m.Get("Foo")
assert.True(t, ok)
assert.Equal(t, v, "bar2")
}
49 changes: 49 additions & 0 deletions pak/metadata.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package pak

// KV represents single key / value meta data entry
type KV struct {
Key string
Value string
}

// Metadata is a list of key / value pairs
type Metadata struct {
Meta []KV
}

// Reset resest Metadata. Useful for re-using (performance)
func (m *Metadata) Reset() {
m.Meta = m.Meta[:0]
}

// Size returns number of metadata e ntries
func (m *Metadata) Size() int {
return len(m.Meta)
}

// Get returns value for a given key
func (m *Metadata) Get(key string) (string, bool) {
for _, kv := range m.Meta {
if kv.Key == key {
return kv.Value, true
}
}
return "", false
}

// Set sets a value for a given key. Returns true if new value was added,
// false value was updated
func (m *Metadata) Set(k, v string) bool {
for i, kv := range m.Meta {
if kv.Key == k {
m.Meta[i].Value = v
return false
}
}
kv := KV{
Key: k,
Value: v,
}
m.Meta = append(m.Meta, kv)
return true
}
Loading

0 comments on commit 11a3ffc

Please sign in to comment.