diff --git a/.dagger/build.go b/.dagger/build.go index e63c569..c1cafd9 100644 --- a/.dagger/build.go +++ b/.dagger/build.go @@ -17,12 +17,12 @@ import ( // // GOOS: linux, windows, darwin // GOARCH: amd64, arm64 -func (t *Tool) BuildPlatforms(ctx context.Context, +func (m *DataTool) BuildPlatforms( // Snapshot build, skipping goreleaser validations. Useful for building committed git history that isn't tagged with a release, a dirty repo does an "auto-snapshot". // +optional snapshot bool, ) *dagger.Directory { - return dag.Goreleaser(t.Source). + return dag.Goreleaser(m.Source). Build(). With(func(r *dagger.GoreleaserBuild) *dagger.GoreleaserBuild { if snapshot { @@ -40,7 +40,7 @@ func (t *Tool) BuildPlatforms(ctx context.Context, // // GOOS: linux, windows, darwin // GOARCH: amd64, arm64 -func (t *Tool) Build(ctx context.Context, +func (m *DataTool) Build(ctx context.Context, // Build target platform // +optional // +default="linux/amd64" @@ -49,11 +49,11 @@ func (t *Tool) Build(ctx context.Context, // +optional snapshot bool, ) *dagger.File { - return build(ctx, t.Source, platform, snapshot) + return build(m.Source, platform, snapshot) } // Create an image with an ace-dt executable. -func (t *Tool) Image(ctx context.Context, +func (m *DataTool) Image(ctx context.Context, // image version version string, // Build target platform @@ -63,14 +63,14 @@ func (t *Tool) Image(ctx context.Context, ) *dagger.Container { ctr := dag.Container(dagger.ContainerOpts{Platform: platform}). From(imageChainguard). - WithFile("/usr/local/bin/ace-dt", t.Build(ctx, platform, false)). + WithFile("/usr/local/bin/ace-dt", m.Build(ctx, platform, false)). WithEntrypoint([]string{"ace-dt"}). WithWorkdir("/") return withCommonLabels(ctr, version) } // Create and publish a multi-platform image index. -func (t *Tool) ImageIndex(ctx context.Context, +func (m *DataTool) ImageIndex(ctx context.Context, // image version version string, // build platforms @@ -90,7 +90,7 @@ func (t *Tool) ImageIndex(ctx context.Context, p := pool.NewWithResults[*dagger.Container]().WithContext(ctx) for _, platform := range platforms { p.Go(func(ctx context.Context) (*dagger.Container, error) { - img := t.Image(ctx, version, platform). + img := m.Image(ctx, version, platform). WithLabel("org.opencontainers.image.url", imgURL). WithLabel("org.opencontainers.image.source", "https://github.com/act3-ai/data-tool") return img, nil @@ -116,7 +116,7 @@ func (t *Tool) ImageIndex(ctx context.Context, result.WriteString(fmt.Sprintf("%s\n", dgstRef)) if len(extraTags) > 0 { - _, err := dag.Release(t.Source).AddTags(ctx, taggedRef, extraTags) + _, err := dag.Release(nil).AddTags(ctx, taggedRef, extraTags) if err != nil { return result.String(), fmt.Errorf("adding extra tags to image: %w", err) } @@ -127,7 +127,7 @@ func (t *Tool) ImageIndex(ctx context.Context, return result.String(), nil } -func build(ctx context.Context, +func build( src *dagger.Directory, platform dagger.Platform, // snapshot build, skip goreleaser validations diff --git a/.dagger/generate.go b/.dagger/generate.go index 5633183..0e5212d 100644 --- a/.dagger/generate.go +++ b/.dagger/generate.go @@ -13,45 +13,44 @@ const ( cliDocsPath = "docs/cli" ) -// Run all auto generators: CLI docs, API docs, and go generate -func (t *Tool) GenAll(ctx context.Context) *dagger.Directory { - return dag.Directory(). - WithDirectory(cliDocsPath, t.CLIDocs(ctx)). - WithDirectory(apiDocsPath, t.APIDocs()). - WithDirectory(pkgPath, t.Generate()) -} - // Generate CLI documentation. -func (t *Tool) CLIDocs(ctx context.Context) *dagger.Directory { - acedt := t.Build(ctx, "linux/amd64", true) +// +generate +func (m *DataTool) CLIDocs(ctx context.Context) *dagger.Changeset { + acedt := m.Build(ctx, "linux/amd64", true) - return dag.Go(). - WithSource(t.Source). - Container(). + return dag.Container(). + WithDirectory("/src", m.Source). + WithWorkdir("/src"). WithFile("/usr/local/bin/ace-dt", acedt). WithExec([]string{"ace-dt", "gendocs", "md", "--only-commands", cliDocsPath}). - Directory(cliDocsPath) + Directory("/src"). + Changes(m.Source) } // Generate API documentation. -func (t *Tool) APIDocs() *dagger.Directory { - return dag.Go(). - WithSource(t.Source). - Exec([]string{"go", "install", goCrdRefDocs}). - WithExec([]string{"crd-ref-docs", "--config=apidocs.yaml", "--renderer=markdown", +// +generate +func (m *DataTool) APIDocs() *dagger.Changeset { + return dag.Go(dagger.GoOpts{ + Source: m.Source, + }). + Env(). + WithExec([]string{"go", "tool", "crd-ref-docs", "--config=apidocs.yaml", "--renderer=markdown", fmt.Sprintf("--source-path=%s/", pkgPath), fmt.Sprintf("--output-path=%s/", apiDocsPath), }). WithoutFile(filepath.Join(apiDocsPath, "out.md")). // TODO: Necessary? - Directory(apiDocsPath) + Directory("/app"). + Changes(m.Source) } // Generate pkg/apis with controller-gen. -func (t *Tool) Generate() *dagger.Directory { - return dag.Go(). - WithSource(t.Source). - WithEnvVariable("GOBIN", "/work/src/tool"). - Exec([]string{"go", "install", goControllerGen}). +// +generate +func (m *DataTool) Generate() *dagger.Changeset { + return dag.Go(dagger.GoOpts{ + Source: m.Source, + }). + Env(). WithExec([]string{"go", "generate", "./..."}). - Directory(pkgPath) + Directory("/app"). + Changes(m.Source) } diff --git a/.dagger/lint.go b/.dagger/lint.go index 8110552..69dc2ef 100644 --- a/.dagger/lint.go +++ b/.dagger/lint.go @@ -10,9 +10,9 @@ import ( ) // Run linters. -func (t *Tool) Lint() *Lint { +func (m *DataTool) Lint() *Lint { return &Lint{ - Source: t.Source, + Source: m.Source, } } diff --git a/.dagger/main.go b/.dagger/main.go index 709d9aa..c4f41e7 100644 --- a/.dagger/main.go +++ b/.dagger/main.go @@ -20,21 +20,16 @@ import ( const ( // images - imageGitCliff = "docker.io/orhunp/git-cliff:2.8.0" imageGrype = "anchore/grype:latest" imageSyft = "anchore/syft:latest" imageRegistry = "docker.io/library/registry:3.0.0-rc.3" imageTelemetry = "ghcr.io/act3-ai/data-telemetry/slim:latest" imageChainguard = "cgr.dev/chainguard/static" imagePostgres = "postgres:17-alpine" - imageGoReleaser = "ghcr.io/goreleaser/goreleaser:v2.8.2" - - // go tools - goControllerGen = "sigs.k8s.io/controller-tools/cmd/controller-gen@v0.17.2" - goCrdRefDocs = "github.com/elastic/crd-ref-docs@v0.1.0" + imageGoReleaser = "ghcr.io/goreleaser/goreleaser:v2.15.3" ) -type Tool struct { +type DataTool struct { // source code directory Source *dagger.Directory @@ -48,40 +43,40 @@ func New( // top level source code directory // +defaultPath="/" src *dagger.Directory, -) *Tool { - return &Tool{ +) *DataTool { + return &DataTool{ Source: src, RegistryConfig: dag.RegistryConfig(), } } // Add credentials for a registry. -func (t *Tool) WithRegistryAuth( +func (m *DataTool) WithRegistryAuth( // registry's hostname address string, // username in registry username string, // password or token for registry secret *dagger.Secret, -) *Tool { - t.RegistryConfig = t.RegistryConfig.WithRegistryAuth(address, username, secret) - return t +) *DataTool { + m.RegistryConfig = m.RegistryConfig.WithRegistryAuth(address, username, secret) + return m } // Removes credentials for a registry. -func (t *Tool) WithoutRegistryAuth( +func (m *DataTool) WithoutRegistryAuth( // registry's hostname address string, -) *Tool { - t.RegistryConfig = t.RegistryConfig.WithoutRegistryAuth(address) - return t +) *DataTool { + m.RegistryConfig = m.RegistryConfig.WithoutRegistryAuth(address) + return m } // Add netrc credentials for a private git repository. -func (t *Tool) WithNetrc( +func (m *DataTool) WithNetrc( // NETRC credentials netrc *dagger.Secret, -) *Tool { - t.Netrc = netrc - return t +) *DataTool { + m.Netrc = netrc + return m } diff --git a/.dagger/release.go b/.dagger/release.go index 2254e0b..ccb9cde 100644 --- a/.dagger/release.go +++ b/.dagger/release.go @@ -15,74 +15,24 @@ const ( repo = "act3-ai/data-tool" ) -// Run release steps. -func (t *Tool) Release() *Releaser { - return &Releaser{ - Tool: t, - } -} - -// Releaser provides utilties for preparing and publishing releases -// with git-cliff. -type Releaser struct { - Tool *Tool -} - -// Run linters, unit, functional, and integration tests. -func (r *Releaser) Check(ctx context.Context) (string, error) { - err := r.diffGenAll(ctx) - if err != nil { - return "", err - } - - // lint, unit test - _, err = dag.Release(r.Tool.Source). - Go(). - Check(ctx, - dagger.ReleaseGolangCheckOpts{ - UnitTestBase: dag.Go(). - WithSource(r.Tool.Source). - Container(). - WithExec([]string{"apt", "update"}). - WithExec([]string{"apt", "install", "-y", "git-lfs"}), - }, - ) - if err != nil { - return "", fmt.Errorf("running linters and unit tests: %w", err) - } - - // functional test - _, err = r.Tool.Test().Functional(ctx) - if err != nil { - return "", fmt.Errorf("running functional tests: %w", err) - } - - // integration test - // _, err = r.Tool.Test().Integration(ctx) - // if err != nil { - // return "", fmt.Errorf("running integration tests: %w", err) - // } - - return "Successfully passed linters and tests", nil -} - // Update the version, changelog, and release notes. -func (r *Releaser) Prepare(ctx context.Context, - // ignore git status checks - // +optional - ignoreError bool, +func (m *DataTool) PrepareRelease(ctx context.Context, + // Git reference to release (can use "." for the current commit) + // +defaultPath="." + gitRef *dagger.GitRef, // release with a specific version // +optional version string, -) (*dagger.Directory, error) { - var err error +) (*dagger.Changeset, error) { + release := dag.Release(gitRef) - targetVersion := version - if targetVersion == "" { - targetVersion, err = r.Version(ctx) + // compute the version if not provided + if version == "" { + v, err := release.Version(ctx) if err != nil { - return nil, fmt.Errorf("resolving release target version: %w", err) + return nil, err } + version = v } // Note: Changes to existing or inclusions of additional image references @@ -90,18 +40,15 @@ func (r *Releaser) Prepare(ctx context.Context, b := &strings.Builder{} b.WriteString("| Images |\n") b.WriteString("| ---------------------------------------------------- |\n") - fmt.Fprintf(b, "| %s/%s:%s |\n\n", reg, repo, targetVersion) + fmt.Fprintf(b, "| %s/%s:%s |\n\n", reg, repo, version) - return dag.Release(r.Tool.Source). - Prepare(dagger.ReleasePrepareOpts{ - Version: targetVersion, - ExtraNotes: b.String(), - IgnoreError: ignoreError}, - ), nil + return release.Prepare(version, dagger.ReleasePrepareOpts{ + ExtraNotes: b.String(), + }), nil } // Create release and publish artifacts. This should already be tagged. -func (r *Releaser) Publish(ctx context.Context, +func (m *DataTool) PublishReleasen(ctx context.Context, // github API token token *dagger.Secret, // commit ssh private key @@ -115,7 +62,7 @@ func (r *Releaser) Publish(ctx context.Context, // +optional latest bool, ) (string, error) { - version, err := r.Tool.Source.File("VERSION").Contents(ctx) + version, err := m.Source.File("VERSION").Contents(ctx) if err != nil { return "", err } @@ -124,7 +71,7 @@ func (r *Releaser) Publish(ctx context.Context, notesPath := filepath.Join("releases", vVersion+".md") imagePlatforms := []dagger.Platform{"linux/amd64", "linux/arm64"} - _, err = dag.Goreleaser(r.Tool.Source, dagger.GoreleaserOpts{Version: "v2.9"}). + _, err = dag.Goreleaser(m.Source, dagger.GoreleaserOpts{Version: "v2.9"}). // env vars defined in .goreleaser.yaml WithSecretVariable("GITHUB_TOKEN", token). WithSecretVariable("SSH_PRIVATE_KEY", sshPrivateKey). @@ -133,57 +80,21 @@ func (r *Releaser) Publish(ctx context.Context, WithEnvVariable("RELEASE_LATEST", strconv.FormatBool(latest)). Release(). WithFailFast(). - WithNotes(r.Tool.Source.File(notesPath)). + WithNotes(m.Source.File(notesPath)). Run(ctx) if err != nil { return "", fmt.Errorf("creating release: %w", err) } regRepo := path.Join("%s/%s", reg, repo) - extraTags, err := dag.Release(r.Tool.Source).ExtraTags(ctx, regRepo, vVersion) + extraTags, err := dag.Release(nil).ExtraTags(ctx, regRepo, vVersion) if err != nil { return "", fmt.Errorf("resolving extra image tags: %w", err) } - _, err = r.Tool.ImageIndex(ctx, vVersion, imagePlatforms, regRepo, extraTags) + _, err = m.ImageIndex(ctx, vVersion, imagePlatforms, regRepo, extraTags) if err != nil { return "", fmt.Errorf("publishing image index: %w", err) } return "Successfully created release and uploaded images", nil } - -// Generate the next version from conventional commit messages (see cliff.toml). Includes 'v' prefix. -func (r *Releaser) Version(ctx context.Context) (string, error) { - targetVersion, err := dag.GitCliff(r.Tool.Source). - BumpedVersion(ctx) - if err != nil { - return "", fmt.Errorf("resolving release target version: %w", err) - } - - return strings.TrimSpace(targetVersion), err -} - -// diffGenAll runs all auto generators, comparing it's output to what currently exists in the source. -func (r *Releaser) diffGenAll(ctx context.Context) error { - existing := dag.Directory(). - WithDirectory(cliDocsPath, r.Tool.Source.Directory(cliDocsPath)). - WithDirectory(apiDocsPath, r.Tool.Source.Directory(apiDocsPath).Filter(dagger.DirectoryFilterOpts{Exclude: []string{"schemas/"}})). - WithDirectory(pkgPath, r.Tool.Source.Directory(pkgPath)) - - regen := r.Tool.GenAll(ctx) - - existingDgst, err := existing.Digest(ctx) - if err != nil { - return err - } - - regenDgst, err := regen.Digest(ctx) - if err != nil { - return err - } - - if existingDgst != regenDgst { - return fmt.Errorf("found changes from running auto generators, please run 'dagger call gen-all export --path=.'") - } - return nil -} diff --git a/.dagger/security.go b/.dagger/security.go index 5b5bcf3..d3bd880 100644 --- a/.dagger/security.go +++ b/.dagger/security.go @@ -6,7 +6,7 @@ import ( ) // Use ace-dt to perform a vulnerability scan on a list of OCI artifacts. -func (t *Tool) Scan(ctx context.Context, +func (m *DataTool) Scan(ctx context.Context, // Path to OCI artifact list sources *dagger.File, ) (string, error) { @@ -14,7 +14,7 @@ func (t *Tool) Scan(ctx context.Context, From(imageGrype). File("/grype") - grypeDB := t.GrypeDB(ctx) + grypeDB := m.GrypeDB() syft := dag.Container(). From(imageSyft). @@ -24,9 +24,9 @@ func (t *Tool) Scan(ctx context.Context, sourcePath := "artifacts.txt" return dag.Container(). - WithMountedSecret("/root/.docker/config.json", t.RegistryConfig.Secret()). + WithMountedSecret("/root/.docker/config.json", m.RegistryConfig.Secret()). From("cgr.dev/chainguard/bash"). - WithFile("/usr/local/bin/ace-dt", build(ctx, t.Source, "linux/amd64", false)). + WithFile("/usr/local/bin/ace-dt", build(m.Source, "linux/amd64", false)). WithFile("/usr/local/bin/grype", grype). WithFile("/usr/local/bin/syft", syft). WithFile(sourcePath, sources). @@ -40,7 +40,7 @@ func (t *Tool) Scan(ctx context.Context, } // Download the Grype vulnerability database -func (t *Tool) GrypeDB(ctx context.Context) *dagger.Directory { +func (m *DataTool) GrypeDB() *dagger.Directory { const cachePath = "/cache/grype" return dag.Container(). diff --git a/.dagger/test.go b/.dagger/test.go index 14f1e80..a343d94 100644 --- a/.dagger/test.go +++ b/.dagger/test.go @@ -16,11 +16,11 @@ const ( ) // Run tests. -func (t *Tool) Test() *Test { +func (m *DataTool) Test() *Test { return &Test{ - Source: t.Source, - Netrc: t.Netrc, - RegistryConfig: t.RegistryConfig, + Source: m.Source, + Netrc: m.Netrc, + RegistryConfig: m.RegistryConfig, } } @@ -53,9 +53,10 @@ func (t *Test) All(ctx context.Context) (string, error) { // Run unit tests. func (t *Test) Unit(ctx context.Context) (string, error) { - return dag.Go(). - WithSource(t.Source). - Container(). + return dag.Go(dagger.GoOpts{ + Source: t.Source, + }). + Env(). WithExec([]string{"apt", "update"}). WithExec([]string{"apt", "install", "-y", "git-lfs"}). WithExec([]string{"go", "test", "./..."}). @@ -98,9 +99,10 @@ func (t *Test) Functional(ctx context.Context) (string, error) { return "", err } acedtConfigPath := "ace-dt-config.yaml" - results, err := dag.Go(). - WithSource(t.Source). - Container(). + results, err := dag.Go(dagger.GoOpts{ + Source: t.Source, + }). + Env(). // dependency for ace-dt git tests WithExec([]string{"apt", "update"}). WithExec([]string{"apt", "install", "-y", "git-lfs"}). @@ -158,7 +160,7 @@ func (t *Test) Integration(ctx context.Context) (string, error) { return "", err } - acedt := build(ctx, t.Source, "linux/amd64", true) + acedt := build(t.Source, "linux/amd64", true) originalBottleRef := "ghcr.io/act3-ai/data-tool/bottles/mnist:v1.6" bottleID, err := dag.Wolfi(). @@ -230,9 +232,10 @@ func (t *Test) Integration(ctx context.Context) (string, error) { // Run benchmark tests. func (t *Test) Bench(ctx context.Context) (string, error) { - return dag.Go(). - WithSource(t.Source). - Container(). + return dag.Go(dagger.GoOpts{ + Source: t.Source, + }). + Env(). WithExec([]string{"go", "test", "./...", "-benchmem", "-bench=.", "-run=^$"}). Stdout(ctx) } diff --git a/.gitignore b/.gitignore index 6fdd232..c4e5867 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /bin/* +!/bin/lib.release.sh +!/bin/release.sh /tool /coverage*.* /report.* diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..25c98c8 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,2 @@ +source-path=SCRIPTDIR +external-sources=true diff --git a/bin/lib.release.sh b/bin/lib.release.sh new file mode 100755 index 0000000..d7254b1 --- /dev/null +++ b/bin/lib.release.sh @@ -0,0 +1,236 @@ +#shellcheck shell=bash + +# Bash library for release process helper functions +# The release script that is sourcing this file may override/edit any of the functions/variables to meet their specific needs. + +set -euo pipefail + +function help() { + cat <&2 + fail=1 + fi + done + if [[ "$fail" == 1 ]]; then + exit 1 + fi +} + +# prepare runs linters and unit tests, bumps the version, and generates the changelog. +function prepare() { + # use given version if provided + local args=() + if [[ -n "$explicit_version" ]]; then + args+=("--version" "$explicit_version") + fi + + dagger_prepare "${args[@]}" + + local version + version=$(