[CWS] Compress btfhub json

pkg/security/probe/constantfetch/btfhub.go

@@ -9,10 +9,13 @@
 package constantfetch
 
 import (
+	"bytes"
+	"compress/gzip"
 	_ "embed" // for go:embed
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"runtime"
 	"strings"
 
@@ -31,13 +34,27 @@ var archMapping = map[string]string{
 	"arm64": "arm64",
 }
 
+func decompressByteArray(data []byte) ([]byte, error) {
+	reader, err := gzip.NewReader(bytes.NewBuffer(data))
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+	decompressed, err := io.ReadAll(reader)
+	return decompressed, err
+}
+
 func (f *BTFHubConstantFetcher) fillStore() error {
 	if len(f.inStore) != 0 {
 		return nil
 	}
 
 	var constantsInfos BTFHubConstants
-	if err := json.Unmarshal(btfhubConstants, &constantsInfos); err != nil {
+	decomp, err := decompressByteArray(btfhubConstants)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(decomp, &constantsInfos); err != nil {
 		return err
 	}
 

pkg/security/probe/constantfetch/btfhub/main.go

@@ -12,6 +12,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"cmp"
+	"compress/gzip"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
@@ -42,11 +43,13 @@ func main() {
 	var archiveRootPath string
 	var constantOutputPath string
 	var combineConstants bool
+	var deflate string
 	var cpuPprofPath string
 
 	flag.StringVar(&archiveRootPath, "archive-root", "", "Root path of BTFHub archive")
 	flag.StringVar(&constantOutputPath, "output", "", "Output path for JSON constants")
 	flag.BoolVar(&combineConstants, "combine", false, "Don't read btf files, but read constants")
+	flag.StringVar(&deflate, "deflate", "", "Deflate the json files")
 	flag.StringVar(&cpuPprofPath, "cpu-prof", "", "Path to the CPU profile to generate")
 	flag.Parse()
 
@@ -63,6 +66,11 @@ func main() {
 		defer pprof.StopCPUProfile()
 	}
 
+	if len(deflate) != 0 {
+		gzipDirJSONs(deflate)
+		return
+	}
+
 	if combineConstants {
 		combined, err := combineConstantFiles(archiveRootPath)
 		if err != nil {
@@ -108,6 +116,50 @@ func main() {
 	}
 }
 
+func gzipDirJSONs(dir string) {
+	gzipFile := func(in string, out string) error {
+		fileOut, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+		if err != nil {
+			return err
+		}
+		defer fileOut.Close()
+
+		writer := gzip.NewWriter(fileOut)
+		s, err := os.ReadFile(in)
+		if err != nil {
+			return err
+		}
+		_, err = writer.Write(s)
+		if err != nil {
+			return err
+		}
+		err = writer.Close()
+		return err
+	}
+
+	err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		if filepath.Ext(path) == ".json" {
+			fmt.Println("Compressing", path)
+			err := gzipFile(path, path+".gz")
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		fmt.Printf("Error compressing all the files in the directory: %v. Please fix the causes and run it again\n", err)
+	}
+}
+
 func outputConstants(export *constantfetch.BTFHubConstants, outputPath string) error {
 	fmt.Printf("%d kernels\n", len(export.Kernels))
 	fmt.Printf("%d unique constants\n", len(export.Constants))

pkg/security/probe/constantfetch/btfhub_amd64.go

@@ -10,5 +10,5 @@ package constantfetch
 
 import _ "embed" // for go:embed
 
-//go:embed btfhub/constants_amd64.json
+//go:embed btfhub/constants_amd64.json.gz
 var btfhubConstants []byte

pkg/security/probe/constantfetch/btfhub_arm64.go

@@ -10,5 +10,5 @@ package constantfetch
 
 import _ "embed" // for go:embed
 
-//go:embed btfhub/constants_arm64.json
+//go:embed btfhub/constants_arm64.json.gz
 var btfhubConstants []byte