Make response file name prefix configurable

Stale files can get large and need cleaning up, then it helps to have a descriptive name (rather than just 'tmp')

Author: larry-cdn77

cache/async_cache.go

@@ -22,6 +22,7 @@ type AsyncCache struct {
 
 	MaxPayloadSize     config.ByteSize
 	SharedWithAllUsers bool
+	TmpFilePrefix      string
 }
 
 func (c *AsyncCache) Close() error {
@@ -114,5 +115,6 @@ func NewAsyncCache(cfg config.Cache, maxExecutionTime time.Duration) (*AsyncCach
 		graceTime:           graceTime,
 		MaxPayloadSize:      maxPayloadSize,
 		SharedWithAllUsers:  cfg.SharedWithAllUsers,
+		TmpFilePrefix:       cfg.TmpFilePrefix,
 	}, nil
 }

cache/filesystem_cache_test.go

@@ -194,7 +194,7 @@ func TestCacheClean(t *testing.T) {
 			Query: []byte(fmt.Sprintf("SELECT %d cache clean", i)),
 		}
 		trw := &testResponseWriter{}
-		crw, err := NewTmpFileResponseWriter(trw, testTmpWriterDir)
+		crw, err := NewTmpFileResponseWriter(trw, testTmpWriterDir, DefaultTmpFilePrefix)
 		if err != nil {
 			t.Fatalf("create tmp cache: %s", err)
 		}

cache/tmp_file_response_writer.go

@@ -25,13 +25,18 @@ type TmpFileResponseWriter struct {
 	bw      *bufio.Writer // buffered writer for the temporary file
 }
 
-func NewTmpFileResponseWriter(rw http.ResponseWriter, dir string) (*TmpFileResponseWriter, error) {
+const DefaultTmpFilePrefix = "chproxyTmp"
+
+func NewTmpFileResponseWriter(rw http.ResponseWriter, dir, prefix string) (*TmpFileResponseWriter, error) {
 	_, ok := rw.(http.CloseNotifier)
 	if !ok {
 		return nil, fmt.Errorf("the response writer does not implement http.CloseNotifier")
 	}
+	if prefix == "" {
+		prefix = DefaultTmpFilePrefix
+	}
 
-	f, err := os.CreateTemp(dir, "tmp")
+	f, err := os.CreateTemp(dir, prefix)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create temporary file in %q: %w", dir, err)
 	}

cache/tmp_file_response_writer_test.go

@@ -5,6 +5,8 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -60,7 +62,7 @@ func TestFileCreation(t *testing.T) {
 	files, _ := os.ReadDir(testTmpWriterDir)
 	nbFileBefore := len(files)
 
-	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir)
+	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix)
 	defer tmpFileRespWriter.Close()
 	if err != nil {
 		t.Fatalf("could not initate TmpFileResponseWriter error:%s", err)
@@ -80,7 +82,7 @@ func TestFileRemoval(t *testing.T) {
 	files, _ := os.ReadDir(testTmpWriterDir)
 	nbFileBefore := len(files)
 
-	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir)
+	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix)
 	if err != nil {
 		t.Fatalf("could not initate TmpFileResponseWriter error:%s", err)
 		return
@@ -98,7 +100,7 @@ func TestFileRemoval(t *testing.T) {
 func TestWriteThenReadHeader(t *testing.T) {
 	srw := newFakeResponse()
 
-	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir)
+	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix)
 	defer tmpFileRespWriter.Close()
 	if err != nil {
 		t.Fatalf("could not initate TmpFileResponseWriter error:%s", err)
@@ -129,7 +131,7 @@ func TestWriteThenReadHeader(t *testing.T) {
 func TestWriteThenReadContent(t *testing.T) {
 	srw := newFakeResponse()
 
-	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir)
+	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix)
 	defer tmpFileRespWriter.Close()
 	if err != nil {
 		t.Fatalf("could not initate TmpFileResponseWriter error:%s", err)
@@ -167,7 +169,7 @@ func TestWriteThenReadStatusCode(t *testing.T) {
 	srw := newFakeResponse()
 	expectStatusCode1 := http.StatusOK
 	expectStatusCode2 := 444
-	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir)
+	tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix)
 	defer tmpFileRespWriter.Close()
 	if err != nil {
 		t.Fatalf("could not initate TmpFileResponseWriter error:%s", err)
@@ -188,3 +190,19 @@ func TestWriteThenReadStatusCode(t *testing.T) {
 	}
 
 }
+
+func TestTmpFilePrefixIsUsed(t *testing.T) {
+	const customPrefix = "myCustomPrefix"
+	srw := newFakeResponse()
+
+	w, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, customPrefix)
+	if err != nil {
+		t.Fatalf("could not create TmpFileResponseWriter: %s", err)
+	}
+	defer w.Close()
+
+	name := filepath.Base(w.tmpFile.Name())
+	if !strings.HasPrefix(name, customPrefix) {
+		t.Fatalf("expected tmp file name to start with %q, got %q", customPrefix, name)
+	}
+}

config/config.go

@@ -936,14 +936,18 @@ type Cache struct {
 
 	Redis RedisCacheConfig `yaml:"redis,omitempty"`
 
-	// Catches all undefined fields
-	XXX map[string]interface{} `yaml:",inline"`
-
 	// Maximum total size of request payload for caching
 	MaxPayloadSize ByteSize `yaml:"max_payload_size,omitempty"`
 
 	// Whether a query cached by a user could be used by another user
 	SharedWithAllUsers bool `yaml:"shared_with_all_users,omitempty"`
+
+	// Prefix for temporary files created during response caching.
+	// Defaults to "chproxyTmp" if not set.
+	TmpFilePrefix string `yaml:"tmp_file_prefix,omitempty"`
+
+	// Catches all undefined fields
+	XXX map[string]interface{} `yaml:",inline"`
 }
 
 func (c *Cache) setDefaults() {

config/config_test.go

@@ -31,6 +31,7 @@ var fullConfig = Config{
 			GraceTime:          Duration(20 * time.Second),
 			MaxPayloadSize:     ByteSize(100 << 30),
 			SharedWithAllUsers: false,
+			TmpFilePrefix:      "chproxyTmp",
 		},
 		{
 			Name: "shortterm",
@@ -983,3 +984,21 @@ func TestConfigReplaceEnvVars(t *testing.T) {
 		})
 	}
 }
+
+func TestCacheTmpFilePrefixParsed(t *testing.T) {
+	const input = `
+name: testcache
+mode: file_system
+tmp_file_prefix: chproxyResponseCache_
+file_system:
+  dir: /tmp
+  max_size: 100Mb
+`
+	var c Cache
+	if err := yaml.Unmarshal([]byte(input), &c); err != nil {
+		t.Fatalf("unexpected parse error: %s", err)
+	}
+	if c.TmpFilePrefix != "chproxyResponseCache_" {
+		t.Fatalf("expected TmpFilePrefix %q, got %q", "chproxyResponseCache_", c.TmpFilePrefix)
+	}
+}

config/testdata/full.yml

@@ -33,6 +33,13 @@ caches:
 
     max_payload_size: 100Gb
 
+    # Optional prefix for temporary files created while streaming a response
+    # into the cache. Useful for distinguishing chproxy temp files from other
+    # processes when inspecting the OS temp directory.
+    #
+    # Defaults to "chproxyTmp".
+    tmp_file_prefix: chproxyTmp
+
     # Expiration time for cached responses.
     expire: 1h
 

docs/src/content/docs/configuration/caching.md

@@ -63,6 +63,22 @@ User Y will get the cached response from user X's query.
 Since 1.20.0, the cache is specific for each user by default since it's better in terms of security.
 It's possible to use the previous behavior by setting the following property of the cache in the config file `shared_with_all_users = true` 
 
+#### Temporary file prefix
+
+While streaming a ClickHouse response into the cache, chproxy writes it to a temporary file in the OS temp directory. The filename starts with a configurable prefix, which defaults to `chproxyTmp`.
+
+Setting a custom prefix is useful when you need to distinguish chproxy temp files from those of other processes, for example in monitoring or cleanup scripts:
+
+```yaml
+caches:
+  - name: my-cache
+    mode: file_system
+    tmp_file_prefix: chproxyResponseCache_
+    file_system:
+      dir: /path/to/cache
+      max_size: 100Mb
+```
+
 #### Detecting Cache Hits
 
 `Chproxy` will respond with an `X-Cache` header with a value of `HIT` if it returned a response from either the local or the distributed cache. Otherwise `X-Cache` will be set to `MISS`. 

proxy.go

@@ -418,7 +418,7 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
 
 	// The response wasn't found in the cache.
 	// Request it from clickhouse.
-	tmpFileRespWriter, err := cache.NewTmpFileResponseWriter(srw, os.TempDir())
+	tmpFileRespWriter, err := cache.NewTmpFileResponseWriter(srw, os.TempDir(), userCache.TmpFilePrefix)
 	if err != nil {
 		err = fmt.Errorf("%s: %w; query: %q", s, err, q)
 		respondWith(srw, err, http.StatusInternalServerError)