diff --git a/.github/workflows/rclone-tests.yaml b/.github/workflows/rclone-tests.yaml new file mode 100644 index 0000000..0438973 --- /dev/null +++ b/.github/workflows/rclone-tests.yaml @@ -0,0 +1,23 @@ +name: rclone tests + +on: + push: + paths: + - 'rclone/**' + pull_request: + paths: + - 'rclone/**' + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - uses: actions/setup-go@v5 + with: + go-version-file: rclone/go.mod + + - name: test + working-directory: rclone + run: go test ./... diff --git a/rclone/connector_test.go b/rclone/connector_test.go new file mode 100644 index 0000000..71ca794 --- /dev/null +++ b/rclone/connector_test.go @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2026 Omar Polo + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +package rclone + +import ( + "bytes" + "io" + "io/fs" + "path" + "sort" + "strings" + "testing" + "time" + + "github.com/PlakarKorp/kloset/connectors" + "github.com/PlakarKorp/kloset/connectors/storage" + "github.com/PlakarKorp/kloset/objects" + rclonefs "github.com/rclone/rclone/fs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + _ "github.com/rclone/rclone/backend/memory" // register memory backend +) + +func newRclone(t *testing.T) *Rclone { + t.Helper() + name := strings.ReplaceAll(t.Name(), "/", "-") + params := map[string]string{ + "location": "rclone://" + name, + "rclone_type": "memory", + } + opts := &connectors.Options{ + Hostname: "localhost", + OperatingSystem: "test", + Architecture: "amd64", + MaxConcurrency: 1, + } + r, err := New(t.Context(), opts, "rclone", params) + require.NoError(t, err) + return r +} + +func putFile(t *testing.T, r *Rclone, relpath, content string) { + t.Helper() + body := []byte(content) + _, err := r.fs.Put(t.Context(), bytes.NewReader(body), &objectinfo{&connectors.Record{ + Pathname: relpath, + FileInfo: objects.FileInfo{ + Lname: path.Base(relpath), + Lsize: int64(len(body)), + Lmode: 0600, + LmodTime: time.Now(), + }, + }}) + require.NoError(t, err) +} + +// runImporter runs r.Import in a goroutine and returns the records channel. +// The caller must drain records; call the returned wait function afterward to +// obtain the Import error once the channel is closed. +func runImporter(t *testing.T, r *Rclone) (<-chan *connectors.Record, func() error) { + t.Helper() + records := make(chan *connectors.Record, 16) + done := make(chan error, 1) + go func() { + done <- r.Import(t.Context(), records, nil) + }() + return records, func() error { return <-done } +} + +// runExporter runs r.Export in a goroutine and returns the results channel. +// The caller must send all records then close records; drain results and call +// the returned wait function afterward to obtain the Export error. +func runExporter(t *testing.T, r *Rclone, records <-chan *connectors.Record) (<-chan *connectors.Result, func() error) { + t.Helper() + results := make(chan *connectors.Result) + done := make(chan error, 1) + go func() { + done <- r.Export(t.Context(), records, results) + }() + return results, func() error { return <-done } +} + +func TestMetadata(t *testing.T) { + r := newRclone(t) + assert.Equal(t, "memory", r.Type()) + assert.Equal(t, "localhost", r.Origin()) + assert.NotEmpty(t, r.Root()) +} + +func TestPing(t *testing.T) { + r := newRclone(t) + // seed a file so the memory backend root exists + putFile(t, r, "ping", "") + require.NoError(t, r.Ping(t.Context())) +} + +func TestImport(t *testing.T) { + r := newRclone(t) + + putFile(t, r, "alpha.txt", "hello alpha") + putFile(t, r, "sub/beta.txt", "hello beta") + + records, wait := runImporter(t, r) + + var paths []string + for rec := range records { + if rec.Err == nil { + paths = append(paths, rec.Pathname) + } + } + require.NoError(t, wait()) + + base := r.Root() + sort.Strings(paths) + assert.Contains(t, paths, path.Join(base, "alpha.txt")) + assert.Contains(t, paths, path.Join(base, "sub/beta.txt")) +} + +func TestImportFileContent(t *testing.T) { + r := newRclone(t) + + putFile(t, r, "hello.txt", "world") + + records, wait := runImporter(t, r) + + base := r.Root() + target := path.Join(base, "hello.txt") + var found bool + for rec := range records { + if rec.Pathname != target { + continue + } + require.NotNil(t, rec.Reader) + got, err := io.ReadAll(rec.Reader) + rec.Reader.Close() + require.NoError(t, err) + assert.Equal(t, "world", string(got)) + found = true + } + require.NoError(t, wait()) + if !found { + t.Fatalf("record %s not found", target) + } +} + +func TestExport(t *testing.T) { + r := newRclone(t) + + records := make(chan *connectors.Record) + results, wait := runExporter(t, r, records) + + go func() { + records <- &connectors.Record{ + Pathname: "outdir", + FileInfo: objects.FileInfo{ + Lname: "outdir", + Lmode: 0700 | fs.ModeDir, + }, + } + records <- &connectors.Record{ + Pathname: "outdir/file.txt", + FileInfo: objects.FileInfo{ + Lname: "file.txt", + Lsize: 5, + Lmode: 0600, + }, + Reader: io.NopCloser(strings.NewReader("hello")), + } + close(records) + }() + + for res := range results { + require.NoError(t, res.Err) + } + require.NoError(t, wait()) + + // verify the file landed in rclone FS + obj, err := r.fs.NewObject(t.Context(), "outdir/file.txt") + require.NoError(t, err) + rd, err := obj.Open(t.Context()) + require.NoError(t, err) + defer rd.Close() + got, err := io.ReadAll(rd) + require.NoError(t, err) + assert.Equal(t, "hello", string(got)) +} + +func TestExportSkipsXattr(t *testing.T) { + r := newRclone(t) + + records := make(chan *connectors.Record) + results, wait := runExporter(t, r, records) + + go func() { + records <- &connectors.Record{ + Pathname: "file.txt", + IsXattr: true, + FileInfo: objects.FileInfo{Lname: "file.txt", Lmode: 0600}, + } + close(records) + }() + + for res := range results { + require.NoError(t, res.Err) + } + require.NoError(t, wait()) + + _, err := r.fs.NewObject(t.Context(), "file.txt") + assert.ErrorIs(t, err, rclonefs.ErrorObjectNotFound) +} + +func TestExportSkipsRoot(t *testing.T) { + r := newRclone(t) + + records := make(chan *connectors.Record) + results, wait := runExporter(t, r, records) + + go func() { + records <- &connectors.Record{ + Pathname: "/", + FileInfo: objects.FileInfo{Lname: "/", Lmode: 0700 | fs.ModeDir}, + } + close(records) + }() + + for res := range results { + require.NoError(t, res.Err) + } + require.NoError(t, wait()) +} + + +func TestStorageCreateOpen(t *testing.T) { + r := newRclone(t) + + conf := []byte(`{"version":1}`) + require.NoError(t, r.Create(t.Context(), conf)) + + got, err := r.Open(t.Context()) + require.NoError(t, err) + assert.Equal(t, conf, got) +} + +func TestStorageCreateAlreadyExists(t *testing.T) { + r := newRclone(t) + + require.NoError(t, r.Create(t.Context(), []byte("config"))) + err := r.Create(t.Context(), []byte("config")) + require.Error(t, err) + assert.Contains(t, err.Error(), "already initialized") +} + +func TestStoragePutGetRoundtrip(t *testing.T) { + resources := []storage.StorageResource{ + storage.StorageResourcePackfile, + storage.StorageResourceState, + storage.StorageResourceLock, + } + + for _, res := range resources { + res := res + t.Run(res.String(), func(t *testing.T) { + r := newRclone(t) + mac := objects.RandomMAC() + + data := []byte("payload data") + n, err := r.Put(t.Context(), res, mac, bytes.NewReader(data)) + require.NoError(t, err) + assert.Equal(t, int64(len(data)), n) + + rd, err := r.Get(t.Context(), res, mac, nil) + require.NoError(t, err) + defer rd.Close() + got, err := io.ReadAll(rd) + require.NoError(t, err) + assert.Equal(t, data, got) + }) + } +} + +func TestStoragePutGetRange(t *testing.T) { + r := newRclone(t) + mac := objects.RandomMAC() + + data := []byte("0123456789") + _, err := r.Put(t.Context(), storage.StorageResourcePackfile, mac, bytes.NewReader(data)) + require.NoError(t, err) + + rg := &storage.Range{Offset: 2, Length: 5} + rd, err := r.Get(t.Context(), storage.StorageResourcePackfile, mac, rg) + require.NoError(t, err) + defer rd.Close() + got, err := io.ReadAll(rd) + require.NoError(t, err) + assert.Equal(t, []byte("23456"), got) +} + +func TestStorageList(t *testing.T) { + r := newRclone(t) + + mac1 := objects.RandomMAC() + mac2 := objects.RandomMAC() + + _, err := r.Put(t.Context(), storage.StorageResourceState, mac1, strings.NewReader("a")) + require.NoError(t, err) + _, err = r.Put(t.Context(), storage.StorageResourceState, mac2, strings.NewReader("b")) + require.NoError(t, err) + + macs, err := r.List(t.Context(), storage.StorageResourceState) + require.NoError(t, err) + assert.Len(t, macs, 2) + assert.Contains(t, macs, mac1) + assert.Contains(t, macs, mac2) +} + +func TestStorageDelete(t *testing.T) { + r := newRclone(t) + mac := objects.RandomMAC() + + _, err := r.Put(t.Context(), storage.StorageResourcePackfile, mac, strings.NewReader("data")) + require.NoError(t, err) + + require.NoError(t, r.Delete(t.Context(), storage.StorageResourcePackfile, mac)) + + macs, err := r.List(t.Context(), storage.StorageResourcePackfile) + require.NoError(t, err) + assert.Empty(t, macs) +} + +func TestStorageUnsupportedResource(t *testing.T) { + r := newRclone(t) + mac := objects.RandomMAC() + + const badRes storage.StorageResource = 99 + + _, err := r.List(t.Context(), badRes) + require.Error(t, err) + + _, err = r.Put(t.Context(), badRes, mac, strings.NewReader("x")) + require.Error(t, err) + + _, err = r.Get(t.Context(), badRes, mac, nil) + require.Error(t, err) + + err = r.Delete(t.Context(), badRes, mac) + require.Error(t, err) +} + +func TestStorageMode(t *testing.T) { + r := newRclone(t) + mode, err := r.Mode(t.Context()) + require.NoError(t, err) + assert.Equal(t, storage.ModeRead|storage.ModeWrite, mode) +} + +func TestStorageSize(t *testing.T) { + r := newRclone(t) + size, err := r.Size(t.Context()) + require.NoError(t, err) + assert.Equal(t, int64(-1), size) +}