diff --git a/pkg/api/core/image/cache.go b/pkg/api/core/image/cache.go index 79abe7106..cb9e1e20e 100644 --- a/pkg/api/core/image/cache.go +++ b/pkg/api/core/image/cache.go @@ -17,7 +17,6 @@ package image import ( "encoding/json" - "fmt" "os" "strings" @@ -156,10 +155,7 @@ func (c *Cache) All(fn func(CacheResult)) { } for key := range c.store.Keys(nil) { - val, err := c.store.Read(key) - if err != nil { - panic(fmt.Sprintf("key %s had no value", key)) - } + val, _ := c.store.Read(key) fn(CacheResult{key: key, value: string(val)}) } } diff --git a/pkg/api/core/image/create.go b/pkg/api/core/image/create.go new file mode 100644 index 000000000..5e3316692 --- /dev/null +++ b/pkg/api/core/image/create.go @@ -0,0 +1,81 @@ +// Copyright © 2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package image + +import ( + "io" + "os" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/pkg/errors" +) + +func imageFromTar(imagename string, r io.Reader) (name.Reference, v1.Image, error) { + newRef, err := name.ParseReference(imagename) + if err != nil { + return nil, nil, err + } + + layer, err := tarball.LayerFromReader(r) + if err != nil { + return nil, nil, err + } + + img, err := mutate.Append(empty.Image, mutate.Addendum{ + Layer: layer, + History: v1.History{ + CreatedBy: "luet", + Comment: "Custom image", + }, + }) + if err != nil { + return nil, nil, err + } + + return newRef, img, nil +} + +// CreateTar a imagetarball from a standard tarball +func CreateTar(srctar, dstimageTar, imagename string) error { + f, err := os.Open(srctar) + if err != nil { + return errors.Wrap(err, "Cannot open "+srctar) + } + defer f.Close() + + return CreateTarReader(f, dstimageTar, imagename) +} + +// CreateTarReader a imagetarball from a standard tarball +func CreateTarReader(r io.Reader, dstimageTar, imagename string) error { + dstFile, err := os.Create(dstimageTar) + if err != nil { + return errors.Wrap(err, "Cannot create "+dstimageTar) + } + defer dstFile.Close() + + newRef, img, err := imageFromTar(imagename, r) + if err != nil { + return err + } + + // NOTE: We might also stream that back to the daemon with daemon.Write(tag, img) + return tarball.Write(newRef, img, dstFile) +} diff --git a/pkg/api/core/image/create_test.go b/pkg/api/core/image/create_test.go new file mode 100644 index 000000000..ab36347ef --- /dev/null +++ b/pkg/api/core/image/create_test.go @@ -0,0 +1,79 @@ +// Copyright © 2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package image_test + +import ( + "os" + "path/filepath" + + . "github.com/mudler/luet/pkg/api/core/image" + "github.com/mudler/luet/pkg/api/core/types" + "github.com/mudler/luet/pkg/api/core/types/artifact" + "github.com/mudler/luet/pkg/compiler/backend" + "github.com/mudler/luet/pkg/helpers/file" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Create", func() { + Context("Creates an OCI image from a standard tar", func() { + It("creates an image which is loadable", func() { + ctx := types.NewContext() + + dst, err := ctx.Config.System.TempFile("dst") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(dst.Name()) + srcTar, err := ctx.Config.System.TempFile("srcTar") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(srcTar.Name()) + + b := backend.NewSimpleDockerBackend(ctx) + + b.DownloadImage(backend.Options{ImageName: "alpine"}) + img, err := b.ImageReference("alpine", false) + Expect(err).ToNot(HaveOccurred()) + + _, dir, err := Extract(ctx, img, false, nil) + Expect(err).ToNot(HaveOccurred()) + + defer os.RemoveAll(dir) + + Expect(file.Touch(filepath.Join(dir, "test"))).ToNot(HaveOccurred()) + Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue()) + + a := artifact.NewPackageArtifact(srcTar.Name()) + a.Compress(dir, 1) + + // Unfortunately there is no other easy way to test this + err = CreateTar(srcTar.Name(), dst.Name(), "testimage") + Expect(err).ToNot(HaveOccurred()) + + b.LoadImage(dst.Name()) + + Expect(b.ImageExists("testimage")).To(BeTrue()) + + img, err = b.ImageReference("testimage", false) + Expect(err).ToNot(HaveOccurred()) + + _, dir, err = Extract(ctx, img, false, nil) + Expect(err).ToNot(HaveOccurred()) + + defer os.RemoveAll(dir) + Expect(file.Exists(filepath.Join(dir, "bin"))).To(BeTrue()) + Expect(file.Exists(filepath.Join(dir, "test"))).To(BeTrue()) + }) + }) +}) diff --git a/pkg/api/core/image/extract.go b/pkg/api/core/image/extract.go index 096473221..a1e8b1fb8 100644 --- a/pkg/api/core/image/extract.go +++ b/pkg/api/core/image/extract.go @@ -32,11 +32,10 @@ import ( "github.com/pkg/errors" ) -// Extract dir: -// -> First extract delta considering the dir -// Afterward create artifact pointing to the dir - -// ExtractDeltaFiles returns an handler to extract files in a list +// ExtractDeltaAdditionsFromImages is a filter that takes two images +// an includes and an excludes list. It computes the delta between the images +// considering the added files only, and applies a filter on them based on the regexes +// in the lists. func ExtractDeltaAdditionsFiles( ctx *types.Context, srcimg v1.Image, @@ -127,6 +126,9 @@ func ExtractDeltaAdditionsFiles( }, nil } +// ExtractFiles returns a filter that extracts files from the given path (if not empty) +// It then filters files by an include and exclude list. +// The list can be regexes func ExtractFiles( ctx *types.Context, prefixPath string, @@ -190,6 +192,9 @@ func ExtractFiles( } } +// ExtractReader perform the extracting action over the io.ReadCloser +// it extracts the files over output. Accepts a filter as an option +// and additional containerd Options func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) { defer reader.Close() @@ -259,6 +264,7 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keep return c, output, nil } +// Extract is just syntax sugar around ExtractReader. It extracts an image into a dir func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) { tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction") if err != nil { @@ -267,6 +273,7 @@ func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *ta return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, keepPerms, filter, opts...) } +// ExtractTo is just syntax sugar around ExtractReader func ExtractTo(ctx *types.Context, img v1.Image, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) { return ExtractReader(ctx, mutate.Extract(img), output, keepPerms, filter, opts...) } diff --git a/pkg/api/core/types/artifact/artifact.go b/pkg/api/core/types/artifact/artifact.go index 9368bb317..03886a189 100644 --- a/pkg/api/core/types/artifact/artifact.go +++ b/pkg/api/core/types/artifact/artifact.go @@ -174,12 +174,6 @@ func (a *PackageArtifact) GetFileName() string { return path.Base(a.Path) } -func (a *PackageArtifact) genDockerfile() string { - return ` -FROM scratch -COPY . /` -} - // CreateArtifactForFile creates a new artifact from the given file func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) { if _, err := os.Stat(s); os.IsNotExist(err) { @@ -211,52 +205,36 @@ func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageAr type ImageBuilder interface { BuildImage(backend.Options) error + LoadImage(path string) error } // GenerateFinalImage takes an artifact and builds a Docker image with its content -func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) { - builderOpts := backend.Options{} - archive, err := ctx.Config.GetSystem().TempDir("archive") +func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) error { + archiveFile, err := os.Open(a.Path) if err != nil { - return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path) - } - defer os.RemoveAll(archive) // clean up - - uncompressedFiles := filepath.Join(archive, "files") - dockerFile := filepath.Join(archive, "Dockerfile") - - if err := os.MkdirAll(uncompressedFiles, os.ModePerm); err != nil { - return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path) - } - - if err := a.Unpack(ctx, uncompressedFiles, keepPerms); err != nil { - return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path) + return errors.Wrap(err, "Cannot open "+a.Path) } + defer archiveFile.Close() - empty, err := fileHelper.DirectoryIsEmpty(uncompressedFiles) + decompressed, err := containerdCompression.DecompressStream(archiveFile) if err != nil { - return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles) + return errors.Wrap(err, "Cannot open "+a.Path) } - // See https://github.com/moby/moby/issues/38039. - // We can't generate FROM scratch empty images. Docker will refuse to export them - // workaround: Inject a .virtual empty file - if empty { - fileHelper.Touch(filepath.Join(uncompressedFiles, ".virtual")) + tempimage, err := ctx.Config.GetSystem().TempFile("tempimage") + if err != nil { + return errors.Wrap(err, "error met while creating tempdir for "+a.Path) } + defer os.RemoveAll(tempimage.Name()) // clean up - data := a.genDockerfile() - if err := ioutil.WriteFile(dockerFile, []byte(data), 0644); err != nil { - return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path) + if err := image.CreateTarReader(decompressed, tempimage.Name(), imageName); err != nil { + return errors.Wrap(err, "could not create image from tar") } - builderOpts = backend.Options{ - ImageName: imageName, - SourcePath: archive, - DockerFileName: dockerFile, - Context: uncompressedFiles, + if err := b.LoadImage(tempimage.Name()); err != nil { + return errors.Wrap(err, "while loading image") } - return builderOpts, b.BuildImage(builderOpts) + return nil } // Compress is responsible to archive and compress to the artifact Path. diff --git a/pkg/api/core/types/artifact/artifact_test.go b/pkg/api/core/types/artifact/artifact_test.go index 11899baad..9a04b4af8 100644 --- a/pkg/api/core/types/artifact/artifact_test.go +++ b/pkg/api/core/types/artifact/artifact_test.go @@ -146,9 +146,8 @@ RUN echo bar > /test2`)) err = a.Compress(tmpdir, 1) Expect(err).ToNot(HaveOccurred()) resultingImage := imageprefix + "foo--1.0" - opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false) + err = a.GenerateFinalImage(ctx, resultingImage, b, false) Expect(err).ToNot(HaveOccurred()) - Expect(opts.ImageName).To(Equal(resultingImage)) Expect(b.ImageExists(resultingImage)).To(BeTrue()) @@ -196,9 +195,8 @@ RUN echo bar > /test2`)) err = a.Compress(tmpdir, 1) Expect(err).ToNot(HaveOccurred()) resultingImage := imageprefix + "foo--1.0" - opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false) + err = a.GenerateFinalImage(ctx, resultingImage, b, false) Expect(err).ToNot(HaveOccurred()) - Expect(opts.ImageName).To(Equal(resultingImage)) Expect(b.ImageExists(resultingImage)).To(BeTrue()) @@ -217,11 +215,7 @@ RUN echo bar > /test2`)) ) Expect(err).ToNot(HaveOccurred()) - Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse()) - content, err := ioutil.ReadFile(filepath.Join(result, ".virtual")) - Expect(err).ToNot(HaveOccurred()) - - Expect(string(content)).To(Equal("")) + Expect(fileHelper.DirectoryIsEmpty(result)).To(BeTrue()) }) It("Retrieves uncompressed name", func() { diff --git a/pkg/compiler/backend.go b/pkg/compiler/backend.go index 26e66c9d8..b76f8e3f4 100644 --- a/pkg/compiler/backend.go +++ b/pkg/compiler/backend.go @@ -26,6 +26,7 @@ func NewBackend(ctx *types.Context, s string) (CompilerBackend, error) { type CompilerBackend interface { BuildImage(backend.Options) error ExportImage(backend.Options) error + LoadImage(string) error RemoveImage(backend.Options) error ImageDefinitionToTar(backend.Options) error diff --git a/pkg/compiler/backend/simpledocker.go b/pkg/compiler/backend/simpledocker.go index d8eec643a..4e7fbd874 100644 --- a/pkg/compiler/backend/simpledocker.go +++ b/pkg/compiler/backend/simpledocker.go @@ -71,6 +71,17 @@ func (s *SimpleDocker) CopyImage(src, dst string) error { return nil } +func (s *SimpleDocker) LoadImage(path string) error { + s.ctx.Debug(":whale: Loading image:", path) + cmd := exec.Command("docker", "load", "-i", path) + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrap(err, "Failed loading image: "+string(out)) + } + s.ctx.Success(":whale: Loaded image:", path) + return nil +} + func (s *SimpleDocker) DownloadImage(opts Options) error { name := opts.ImageName bus.Manager.Publish(bus.EventImagePrePull, opts) diff --git a/pkg/compiler/backend/simpleimg.go b/pkg/compiler/backend/simpleimg.go index 1a1729bc6..3dadb7c46 100644 --- a/pkg/compiler/backend/simpleimg.go +++ b/pkg/compiler/backend/simpleimg.go @@ -35,6 +35,10 @@ func NewSimpleImgBackend(ctx *types.Context) *SimpleImg { return &SimpleImg{ctx: ctx} } +func (s *SimpleImg) LoadImage(string) error { + return errors.New("Not supported") +} + // TODO: Missing still: labels, and build args expansion func (s *SimpleImg) BuildImage(opts Options) error { name := opts.ImageName diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 8d81c09af..af9dbd02b 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -962,14 +962,14 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool joinImageName := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, overallFp) cs.Options.Context.Info(joinTag, ":droplet: generating image from artifact", joinImageName) - opts, err := a.GenerateFinalImage(cs.Options.Context, joinImageName, cs.Backend, keepPermissions) + err = a.GenerateFinalImage(cs.Options.Context, joinImageName, cs.Backend, keepPermissions) if err != nil { return errors.Wrap(err, "could not create final image") } if cs.Options.Push { cs.Options.Context.Info(joinTag, ":droplet: pushing image from artifact", joinImageName) - if err = cs.Backend.Push(opts); err != nil { - return errors.Wrapf(err, "Could not push image: %s %s", image, opts.DockerFileName) + if err = cs.Backend.Push(backend.Options{ImageName: joinImageName}); err != nil { + return errors.Wrapf(err, "Could not push image: %s", joinImageName) } } cs.Options.Context.Info(joinTag, ":droplet: Consuming image", joinImageName) diff --git a/pkg/installer/repository_docker.go b/pkg/installer/repository_docker.go index 5870e1f58..f141303e4 100644 --- a/pkg/installer/repository_docker.go +++ b/pkg/installer/repository_docker.go @@ -92,8 +92,8 @@ func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDataba } else { l.context.Info("Generating final image", packageImage, "for package ", a.CompileSpec.GetPackage().HumanReadableString()) - if opts, err := a.GenerateFinalImage(l.context, packageImage, l.b, true); err != nil { - return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName) + if err := a.GenerateFinalImage(l.context, packageImage, l.b, true); err != nil { + return errors.Wrap(err, "Failed generating metadata tree"+packageImage) } } if l.imagePush { @@ -125,8 +125,8 @@ func pushImage(ctx *types.Context, b compiler.CompilerBackend, image string, for func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error { d.context.Debug("Generating image", imageTree) - if opts, err := a.GenerateFinalImage(d.context, imageTree, d.b, false); err != nil { - return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName) + if err := a.GenerateFinalImage(d.context, imageTree, d.b, false); err != nil { + return errors.Wrap(err, "Failed generating metadata tree "+imageTree) } if d.imagePush { if err := pushImage(d.context, d.b, imageTree, true); err != nil { diff --git a/pkg/installer/repository_test.go b/pkg/installer/repository_test.go index d57f86bb0..e61276ca8 100644 --- a/pkg/installer/repository_test.go +++ b/pkg/installer/repository_test.go @@ -619,11 +619,8 @@ urls: Expect(a.Unpack(ctx, extracted, false)).ToNot(HaveOccurred()) - Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeFalse()) - content, err := ioutil.ReadFile(filepath.Join(extracted, ".virtual")) - Expect(err).ToNot(HaveOccurred()) + Expect(fileHelper.DirectoryIsEmpty(extracted)).To(BeTrue()) - Expect(string(content)).To(Equal("")) }) It("Searches files", func() { diff --git a/tests/fixtures/caps/pkgC/0.1/build.yaml b/tests/fixtures/caps/pkgC/0.1/build.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/caps/pkgC/0.1/definition.yaml b/tests/fixtures/caps/pkgC/0.1/definition.yaml new file mode 100644 index 000000000..15d09f47e --- /dev/null +++ b/tests/fixtures/caps/pkgC/0.1/definition.yaml @@ -0,0 +1,3 @@ +category: "test" +name: "empty" +version: "0.1" diff --git a/tests/integration/01_simple_docker.sh b/tests/integration/01_simple_docker.sh index 18c8a54f9..0aa54f561 100755 --- a/tests/integration/01_simple_docker.sh +++ b/tests/integration/01_simple_docker.sh @@ -49,8 +49,10 @@ testRepo() { --meta-compression zstd \ --type docker --push-images --force-push) + echo "$createres" + createst=$? - assertEquals 'create repo successfully' "$createst" "0" + assertEquals 'create repo successfully' "0" "$createst" assertContains 'contains image push' "$createres" 'Pushed image: quay.io/mocaccinoos/integration-test:z-test-1.0-2' } diff --git a/tests/integration/35_caps_docker.sh b/tests/integration/35_caps_docker.sh new file mode 100755 index 000000000..e5a046c3d --- /dev/null +++ b/tests/integration/35_caps_docker.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +export LUET_NOLOCK=true + +oneTimeSetUp() { +export tmpdir="$(mktemp -d)" +} + +oneTimeTearDown() { + rm -rf "$tmpdir" +} + +testBuild() { + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + [ "$LUET_BACKEND" == "img" ] && startSkipping + + mkdir $tmpdir/testbuild + luet build -d --tree "$ROOT_DIR/tests/fixtures/caps" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full + buildst=$? + assertTrue 'create package caps 0.1' "[ -e '$tmpdir/testbuild/caps-test-0.1.package.tar.gz' ]" + assertEquals 'builds successfully' "$buildst" "0" +} + +testRepo() { + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + [ "$LUET_BACKEND" == "img" ] && startSkipping + + assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]" + luet create-repo --tree "$ROOT_DIR/tests/fixtures/caps" \ + --output $TEST_DOCKER_IMAGE \ + --packages $tmpdir/testbuild \ + --name "test" \ + --descr "Test Repo" \ + --push-images --force-push \ + --type docker + + createst=$? + assertEquals 'create repo successfully' "$createst" "0" +} + +testConfig() { + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + [ "$LUET_BACKEND" == "img" ] && startSkipping + + mkdir $tmpdir/testrootfs + cat < $tmpdir/luet.yaml +general: + debug: true +system: + rootfs: $tmpdir/testrootfs + database_path: "/" + database_engine: "boltdb" +config_from_host: true +repositories: + - name: "main" + type: "docker" + enable: true + urls: + - "$TEST_DOCKER_IMAGE" +EOF + luet config --config $tmpdir/luet.yaml + res=$? + assertEquals 'config test successfully' "$res" "0" +} + +testInstall() { + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + [ "$LUET_BACKEND" == "img" ] && startSkipping + + $ROOT_DIR/tests/integration/bin/luet install -y --config $tmpdir/luet.yaml test/caps@0.1 test/caps2@0.1 test/empty + installst=$? + assertEquals 'install test successfully' "$installst" "0" + + assertTrue 'package installed file1' "[ -e '$tmpdir/testrootfs/file1' ]" + assertTrue 'package installed file2' "[ -e '$tmpdir/testrootfs/file2' ]" + + assertContains 'caps' "$(getcap $tmpdir/testrootfs/file1)" "cap_net_raw+ep" + assertContains 'caps' "$(getcap $tmpdir/testrootfs/file2)" "cap_net_raw+ep" +} + + +# Load shUnit2. +. "$ROOT_DIR/tests/integration/shunit2"/shunit2 +