diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f79d44f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +/front/node_modules +/vol +/.idea +/data \ No newline at end of file diff --git a/proto/admin/.keep b/.fastad_root similarity index 100% rename from proto/admin/.keep rename to .fastad_root diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index abcf7b3..596067b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,89 +4,107 @@ on: pull_request: branches: - master + +permissions: + contents: read + checks: write + jobs: tests: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - - name: Set up Go 1.18 - uses: actions/setup-go@v2 + - name: Set up Go + uses: actions/setup-go@v5 with: - go-version: 1.18 - - - name: "Cache go deps" - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + go-version-file: 'go.mod' - name: Run tests run: make test - lint-go: + e2e-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + preset: [simple, production] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 + - name: Set up Go + uses: actions/setup-go@v5 with: - version: latest - args: --config .golangci.yml + go-version-file: 'go.mod' - lint-front: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Set up Node 16 - uses: actions/setup-node@v2 - with: - node-version: "16" + - name: Pre-build Docker images + run: | + go run ./cmd/fastad init --game-config tests/e2e/fastad_test.yaml --preset ${{ matrix.preset }} + docker compose -f compose.yml build - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" + - name: Run E2E tests (${{ matrix.preset }} preset) + run: go test -tags "e2e e2e_${{ matrix.preset }}" -v ./tests/e2e/... -timeout 15m - - name: "Cache deps" - uses: actions/cache@v3 - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- + - name: Collect Docker logs on failure + if: failure() + run: | + docker compose -f compose.yml logs --tail=100 || true + docker ps -a || true - - name: Install dependencies - working-directory: front - run: yarn install + lint-go: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 - - name: Run linters - uses: wearerequired/lint-action@v2 + - name: Set up Go + uses: actions/setup-go@v5 with: - eslint: true - eslint_dir: front/src - eslint_args: "--ext .js,.vue,.ts" + go-version-file: 'go.mod' - - name: Check the frontend compiles - working-directory: front - run: yarn build + - name: golangci-lint + uses: golangci/golangci-lint-action@v8 + with: + version: v2.8.0 lint-proto: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup buf - uses: bufbuild/buf-setup-action@v1.7.0 + uses: bufbuild/buf-setup-action@v1.48.0 - name: Lint proto run: make lint-proto + + lint-front: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + package_json_file: 'front/package.json' + + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version-file: 'front/.node-version' + cache: 'pnpm' + cache-dependency-path: 'front/pnpm-lock.yaml' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + working-directory: front + + - name: Lint front + run: make lint-front diff --git a/.gitignore b/.gitignore index 4c8b52f..69fc208 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,13 @@ .idea .DS_Store .vscode +mise.local.toml +__pycache__ +.sccignore +.generated +.env +/compose.yml +/fastad.yaml +.sisyphus +/fastad +/data diff --git a/.golangci.yml b/.golangci.yml index f5c418d..5eea9c6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,51 +1,160 @@ ---- +version: "2" run: - timeout: 5m + concurrency: 4 modules-download-mode: readonly - + tests: true linters: + default: none enable: + - asasalint - asciicheck + - bidichk - bodyclose - - deadcode + - canonicalheader + - containedctx + - copyloopvar - dogsled - durationcheck + - errcheck - errname - - exportloopref + - errorlint - exhaustive + - fatcontext + - forbidigo + - forcetypeassert + - gocheckcompilerdirectives + - gochecknoinits + - goconst - gocritic - - gofmt - - goimports + - godot - gomoddirectives - - gosimple + - gomodguard + - goprintffuncname + - gosmopolitan - govet - - ifshort + - inamedparam - ineffassign - - importas + - intrange + - loggercheck + - makezero + - mirror - misspell + - musttag + - nilerr + - nilnil - noctx + - nolintlint + - nosprintfhostport + - perfsprint - prealloc - predeclared + - protogetter + - reassign - revive + - rowserrcheck + - spancheck + - sqlclosecheck - staticcheck + - testifylint + - testpackage - thelper - tparallel - unconvert - unparam - unused - - varcheck + - usestdlibvars + - wastedassign - whitespace - wrapcheck - -linters-settings: - wrapcheck: - ignoreSigs: - - errors.New - - (context.Context).Err() - - google.golang.org/grpc/status.Error - - google.golang.org/grpc/status.Errorf - - .Read(p []byte) # most Read errors should be passed as-is, e.g. io.EOF. - -issues: - max-issues-per-linter: 0 - max-same-issues: 0 + settings: + asasalint: + use-builtin-exclusions: false + copyloopvar: + check-alias: true + errcheck: + disable-default-exclusions: true + check-type-assertions: true + check-blank: true + exhaustive: + default-signifies-exhaustive: true + forbidigo: + forbid: + - pattern: self + - pattern: this + exclude-godoc-examples: false + godot: + capital: true + gomoddirectives: + exclude-forbidden: true + gomodguard: + blocked: + modules: + - github.com/golang/protobuf: + recommendations: + - google.golang.org/protobuf + - github.com/satori/go.uuid: + recommendations: + - github.com/google/uuid + - github.com/gofrs/uuid: + recommendations: + - github.com/gofrs/uuid/v5 + govet: + disable: + - fieldalignment + - shadow + enable-all: true + inamedparam: + skip-single-param: true + loggercheck: + require-string-key: true + makezero: + always: true + nolintlint: + require-explanation: true + require-specific: true + perfsprint: + strconcat: false + wrapcheck: + ignore-sigs: + - func errors.New(text string) error + - status.Error( + - status.Errorf( + - fmt.Errorf( + - (context.Context).Err() + - (*google.golang.org/grpc/internal/status.Status).Err() + - (github.com/labstack/echo/v4.Context).JSONBlob(code int, b []byte) error + - (github.com/labstack/echo/v4.Context).Blob(code int, contentType string, b []byte) error + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + rules: + - linters: + - goconst + path: (.+)_test\.go +formatters: + enable: + - gofmt + - gofumpt + - gci + settings: + gofmt: + simplify: true + rewrite-rules: + - pattern: 'interface{}' + replacement: 'any' + gofumpt: + extra-rules: true + gci: + sections: + - standard + - default + - prefix(github.com/c4t-but-s4d/fastad) + - blank + - localmodule + custom-order: true + exclusions: + generated: lax diff --git a/Makefile b/Makefile index 733cd9a..b26ead8 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: lint-proto lint-proto: - cd proto && buf lint && buf build + cd proto && buf lint .PHONY: lint-go lint-go: @@ -8,10 +8,15 @@ lint-go: .PHONY: lint-front lint-front: - cd front && yarn lint + cd front && pnpm lint + +.PHONY: goimports +goimports: + gofancyimports fix --local github.com/c4t-but-s4d/fastad -w $(shell find . -type f -name '*.go' -not -path "./pkg/proto/*") .PHONY: proto proto: lint-proto + rm -rf pkg/proto front/src/proto cd proto && buf generate .PHONY: tidy @@ -27,3 +32,22 @@ test-go: .PHONY: test test: test-go + +.PHONY: commit-proto +commit-proto: proto + git add proto + git add pkg/proto + git add front/src/proto + git commit -m "Regenerate proto" + +.PHONY: reset-db +reset-db: + docker compose exec postgres psql -U fastad fastad -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" + go run ./cmd/migrator/main.go init + +.PHONY: migrate-db +migrate-db: + go run ./cmd/migrator/main.go migrate + +.PHONY: db +db: reset-db migrate-db diff --git a/cmd/allinone/main.go b/cmd/allinone/main.go new file mode 100644 index 0000000..b463c4d --- /dev/null +++ b/cmd/allinone/main.go @@ -0,0 +1,172 @@ +package main + +import ( + "fmt" + + "github.com/google/uuid" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + apiImpl "github.com/c4t-but-s4d/fastad/cmd/api/impl" + checkersImpl "github.com/c4t-but-s4d/fastad/cmd/checkers/impl" + dataserviceImpl "github.com/c4t-but-s4d/fastad/cmd/dataservice/impl" + receiverImpl "github.com/c4t-but-s4d/fastad/cmd/receiver/impl" + schedulerImpl "github.com/c4t-but-s4d/fastad/cmd/scheduler/impl" + slacImpl "github.com/c4t-but-s4d/fastad/cmd/slac/impl" + "github.com/c4t-but-s4d/fastad/internal/checkers" + "github.com/c4t-but-s4d/fastad/internal/dataservice" + "github.com/c4t-but-s4d/fastad/internal/handlers" + "github.com/c4t-but-s4d/fastad/internal/receiver" + "github.com/c4t-but-s4d/fastad/internal/scheduler" + "github.com/c4t-but-s4d/fastad/internal/slac" + "github.com/c4t-but-s4d/fastad/pkg/apiwait" + "github.com/c4t-but-s4d/fastad/pkg/baseconfig" + "github.com/c4t-but-s4d/fastad/pkg/config" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/metrics" + "github.com/c4t-but-s4d/fastad/pkg/stop" +) + +type Config struct { + UserAgent string `mapstructure:"user_agent" default:"allinone"` + + MetricsAddress string `mapstructure:"metrics_address" default:":3000"` + IntercomToken string `mapstructure:"intercom_token"` + + Postgres config.Postgres `mapstructure:"postgres"` + Temporal config.Temporal `mapstructure:"temporal"` + DataService dataservice.Config `mapstructure:"data_service"` + Scheduler scheduler.Config `mapstructure:"scheduler"` + Slac slac.Config `mapstructure:"slac"` + API handlers.Config `mapstructure:"api"` + Checkers checkers.Config `mapstructure:"checkers"` + Receiver receiver.Config `mapstructure:"receiver"` +} + +func main() { + defer logging.Init().Close() + + cfg := baseconfig.MustSetupAll(&Config{}, baseconfig.WithEnvPrefix("FASTAD_ALLINONE")) + + setupConfig(cfg) + + runCtx, shutdownCtx, cancel := stop.SetupCtx() + defer cancel() + + g, gctx := errgroup.WithContext(runCtx) + + g.Go(func() error { + if err := dataserviceImpl.Run(gctx, shutdownCtx, &cfg.DataService); err != nil { + return fmt.Errorf("running dataservice: %w", err) + } + return nil + }) + + g.Go(func() error { + if err := slacImpl.Run(gctx, shutdownCtx, &cfg.Slac); err != nil { + return fmt.Errorf("running slac: %w", err) + } + return nil + }) + + if err := apiwait.GRPC(gctx, cfg.DataService.ListenAddress); err != nil { + zap.L().Error("waiting for DataService", zap.Error(err)) + cancel() + return + } + + if err := apiwait.GRPC(gctx, cfg.Slac.ListenAddress); err != nil { + zap.L().Error("waiting for SLAC", zap.Error(err)) + cancel() + return + } + + cfg.API.DataService.Address = cfg.DataService.ListenAddress + cfg.API.SlacAddress = cfg.Slac.ListenAddress + cfg.API.ReceiverAddress = cfg.Receiver.ListenAddress + g.Go(func() error { + if err := apiImpl.Run(gctx, shutdownCtx, &cfg.API); err != nil { + return fmt.Errorf("running api: %w", err) + } + return nil + }) + + if err := apiwait.HTTP(gctx, cfg.API.ListenAddress); err != nil { + zap.L().Error("waiting for API", zap.Error(err)) + cancel() + return + } + + cfg.Receiver.DataService.Address = cfg.DataService.ListenAddress + cfg.Receiver.CentrifugeClient.Address = fmt.Sprintf("ws://%s/centrifuge/websocket", cfg.API.ListenAddress) + g.Go(func() error { + if err := receiverImpl.Run(gctx, shutdownCtx, &cfg.Receiver); err != nil { + return fmt.Errorf("running receiver: %w", err) + } + return nil + }) + + cfg.Checkers.DataService.Address = cfg.DataService.ListenAddress + g.Go(func() error { + if err := checkersImpl.Run(gctx, shutdownCtx, &cfg.Checkers); err != nil { + return fmt.Errorf("running checkers: %w", err) + } + return nil + }) + + cfg.Scheduler.DataService.Address = cfg.DataService.ListenAddress + g.Go(func() error { + if err := schedulerImpl.Run(gctx, shutdownCtx, &cfg.Scheduler); err != nil { + return fmt.Errorf("running scheduler: %w", err) + } + return nil + }) + + g.Go(func() error { + metrics.RunServer(gctx, shutdownCtx, cfg.MetricsAddress) + return nil + }) + + if err := g.Wait(); err != nil { + zap.L().Fatal("all-in-one run failed", zap.Error(err)) + } +} + +func setupConfig(cfg *Config) { + if cfg.IntercomToken == "" { + cfg.IntercomToken = uuid.NewString() + } + + cfg.DataService.IntercomToken = cfg.IntercomToken + cfg.API.IntercomToken = cfg.IntercomToken + cfg.Slac.IntercomToken = cfg.IntercomToken + cfg.Receiver.IntercomToken = cfg.IntercomToken + cfg.Checkers.IntercomToken = cfg.IntercomToken + cfg.Scheduler.IntercomToken = cfg.IntercomToken + + cfg.DataService.Postgres = cfg.Postgres + cfg.Scheduler.Postgres = cfg.Postgres + cfg.Slac.Postgres = cfg.Postgres + cfg.API.Postgres = cfg.Postgres + cfg.Checkers.Postgres = cfg.Postgres + cfg.Receiver.Postgres = cfg.Postgres + + cfg.Scheduler.Temporal = cfg.Temporal + cfg.Checkers.Temporal = cfg.Temporal + + cfg.DataService.Installation = fmt.Sprintf("%s/dataservice", cfg.UserAgent) + cfg.Scheduler.Installation = fmt.Sprintf("%s/scheduler", cfg.UserAgent) + cfg.Slac.Installation = fmt.Sprintf("%s/slac", cfg.UserAgent) + cfg.API.Installation = fmt.Sprintf("%s/api", cfg.UserAgent) + cfg.Checkers.Installation = fmt.Sprintf("%s/checkers", cfg.UserAgent) + cfg.Receiver.Installation = fmt.Sprintf("%s/receiver", cfg.UserAgent) + + cfg.DataService.MetricsAddress = "" + cfg.Scheduler.MetricsAddress = "" + cfg.Slac.MetricsAddress = "" + cfg.API.MetricsAddress = "" + cfg.Receiver.MetricsAddress = "" + + // Checkers are listening on a different port. + cfg.Checkers.MetricsAddress = "" +} diff --git a/cmd/api/impl/run.go b/cmd/api/impl/run.go new file mode 100644 index 0000000..7c89293 --- /dev/null +++ b/cmd/api/impl/run.go @@ -0,0 +1,196 @@ +package impl + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/centrifugal/centrifuge" + "github.com/labstack/echo-contrib/echoprometheus" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + "github.com/c4t-but-s4d/fastad/internal/centutil" + "github.com/c4t-but-s4d/fastad/internal/handlers" + "github.com/c4t-but-s4d/fastad/pkg/clients/gamestate" + "github.com/c4t-but-s4d/fastad/pkg/clients/services" + "github.com/c4t-but-s4d/fastad/pkg/clients/teams" + "github.com/c4t-but-s4d/fastad/pkg/grpcext" + "github.com/c4t-but-s4d/fastad/pkg/httpext" + "github.com/c4t-but-s4d/fastad/pkg/metrics" + gspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/game_state" + servicespb "github.com/c4t-but-s4d/fastad/pkg/proto/data/services" + teamspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/teams" + receiverpb "github.com/c4t-but-s4d/fastad/pkg/proto/receiver" + slacpb "github.com/c4t-but-s4d/fastad/pkg/proto/slac" +) + +func Run(runCtx, shutdownCtx context.Context, cfg *handlers.Config) error { + dataServiceConn, err := grpcext.Dial( + cfg.DataService.Address, + cfg.Installation, + grpcext.AuthDialOptions(cfg.IntercomToken)..., + ) + if err != nil { + return fmt.Errorf("connecting to data service: %w", err) + } + + teamsClient := teams.NewClient(teamspb.NewTeamsServiceClient(dataServiceConn), cfg.Installation) + servicesClient := services.NewClient(servicespb.NewServicesServiceClient(dataServiceConn), cfg.Installation) + gameStateClient := gamestate.NewClient(gspb.NewGameStateServiceClient(dataServiceConn), cfg.Installation) + + receiverConn, err := grpcext.Dial( + cfg.ReceiverAddress, + cfg.Installation, + grpcext.AuthDialOptions(cfg.IntercomToken)..., + ) + if err != nil { + return fmt.Errorf("connecting to receiver: %w", err) + } + receiverClient := receiverpb.NewReceiverServiceClient(receiverConn) + + slacConn, err := grpcext.Dial( + cfg.SlacAddress, + cfg.Installation, + grpcext.AuthDialOptions(cfg.IntercomToken)..., + ) + if err != nil { + return fmt.Errorf("connecting to slac: %w", err) + } + slacClient := slacpb.NewSlacServiceClient(slacConn) + + centConfig := centrifuge.Config{ + Name: cfg.Installation, + LogLevel: centrifuge.LogLevelDebug, + LogHandler: func(entry centrifuge.LogEntry) { + zap.L().Debug( + "centrifuge log", + zap.String("message", entry.Message), + zap.Any("entry", entry.Fields), + ) + }, + } + node, err := centrifuge.New(centConfig) + if err != nil { + return fmt.Errorf("creating centrifuge node: %w", err) + } + + producer := centutil.NewNodeProducer(node, cfg.ScoreboardChannel) + + boardBuilder := handlers.NewBoardBuilder( + teamsClient, + servicesClient, + receiverClient, + slacClient, + producer, + ) + + db := cfg.Postgres.BunDB() + + apiService := handlers.NewService( + cfg, + db, + node, + teamsClient, + servicesClient, + gameStateClient, + receiverClient, + slacClient, + boardBuilder, + ) + + e := echo.New() + e.Use( + echoprometheus.NewMiddleware("http"), + httpext.RequestIDMiddleware(), + middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ + LogMethod: true, + LogURI: true, + LogStatus: true, + LogLatency: true, + LogRemoteIP: true, + LogRoutePath: true, + LogValuesFunc: func(_ echo.Context, v middleware.RequestLoggerValues) error { + logFunc := zap.L().Debug + switch v.Status / 100 { + case 5: + logFunc = zap.L().Error + case 4: + logFunc = zap.L().Warn + } + + logFunc("request", + zap.String("method", v.Method), + zap.String("URI", v.URI), + zap.Int("status", v.Status), + zap.Duration("latency", v.Latency), + zap.String("remote_ip", v.RemoteIP), + zap.String("path", v.RoutePath), + ) + + return nil + }, + }), + middleware.Recover(), + middleware.Gzip(), + middleware.Secure(), // TODO: allow images from external sources for avatars. + middleware.CORSWithConfig(middleware.CORSConfig{ + AllowOrigins: []string{"*"}, + AllowCredentials: true, + UnsafeWildcardOriginWithAllowCredentials: true, + }), + ) + e.GET("/healthcheck", httpext.HealthHandler()) + + e.HTTPErrorHandler = httpext.ErrorHandler() + apiService.RegisterRoutes(e) + apiService.RegisterNode() + + if err := node.Run(); err != nil { + return fmt.Errorf("running centrifuge node: %w", err) + } + + g, gctx := errgroup.WithContext(runCtx) + g.Go(func() error { + boardBuilder.Run(gctx) + return nil + }) + + if cfg.MetricsAddress != "" { + g.Go(func() error { + metrics.RunServer(gctx, shutdownCtx, cfg.MetricsAddress) + return nil + }) + } + + g.Go(func() error { + zap.L().Info("starting api server", zap.String("address", cfg.ListenAddress)) + if err := e.Start(cfg.ListenAddress); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("running server: %w", err) + } + zap.L().Info("api server stopped") + return nil + }) + + g.Go(func() error { + <-gctx.Done() + + zap.L().Info("shutting down api server") + if err := e.Shutdown(shutdownCtx); err != nil { + return fmt.Errorf("shutting down server: %w", err) + } + if err := node.Shutdown(shutdownCtx); err != nil { + return fmt.Errorf("shutting down centrifuge node: %w", err) + } + return nil + }) + + if err := g.Wait(); err != nil { + return fmt.Errorf("waiting for goroutines: %w", err) + } + + return nil +} diff --git a/cmd/api/main.go b/cmd/api/main.go new file mode 100644 index 0000000..3d00a0b --- /dev/null +++ b/cmd/api/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/cmd/api/impl" + "github.com/c4t-but-s4d/fastad/internal/handlers" + "github.com/c4t-but-s4d/fastad/pkg/baseconfig" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/stop" +) + +func main() { + defer logging.Init().Close() + + cfg := baseconfig.MustSetupAll(&handlers.Config{}, baseconfig.WithEnvPrefix("FASTAD_API")) + + runCtx, shutdownCtx, cancel := stop.SetupCtx() + defer cancel() + + if err := impl.Run(runCtx, shutdownCtx, cfg); err != nil { + zap.L().Fatal("api run failed", zap.Error(err)) + } +} diff --git a/cmd/checkers/impl/run.go b/cmd/checkers/impl/run.go new file mode 100644 index 0000000..7930cd2 --- /dev/null +++ b/cmd/checkers/impl/run.go @@ -0,0 +1,148 @@ +package impl + +import ( + "context" + "fmt" + + "go.temporal.io/sdk/activity" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/worker" + "go.temporal.io/sdk/workflow" + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/internal/checkers" + "github.com/c4t-but-s4d/fastad/pkg/clients/gamestate" + "github.com/c4t-but-s4d/fastad/pkg/clients/services" + "github.com/c4t-but-s4d/fastad/pkg/clients/teams" + "github.com/c4t-but-s4d/fastad/pkg/grpcext" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/metrics" + gspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/game_state" + servicespb "github.com/c4t-but-s4d/fastad/pkg/proto/data/services" + teamspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/teams" +) + +func Run(runCtx, shutdownCtx context.Context, cfg *checkers.Config) error { + temporalClientOpts := client.Options{ + HostPort: cfg.Temporal.Address, + Logger: logging.NewTemporalAdapter( + zap.L().Named("checkers_worker"), + ), + } + if cfg.MetricsAddress != "" { + handler, err := metrics.TemporalHandler(cfg.MetricsAddress) + if err != nil { + return fmt.Errorf("creating metrics handler: %w", err) + } + temporalClientOpts.MetricsHandler = handler + } + temporalClient, err := client.Dial(temporalClientOpts) + if err != nil { + return fmt.Errorf("creating temporal client: %w", err) + } + defer temporalClient.Close() + + db := cfg.Postgres.BunDB() + + checkersMetrics := checkers.NewMetrics(cfg.Installation) + checkersController := checkers.NewController(db, checkersMetrics) + + dataServiceConn, err := grpcext.Dial( + cfg.DataService.Address, + cfg.Installation, + grpcext.AuthDialOptions(cfg.IntercomToken)..., + ) + if err != nil { + return fmt.Errorf("dialing data service: %w", err) + } + + teamsClient := teams.NewClient(teamspb.NewTeamsServiceClient(dataServiceConn), cfg.Installation) + servicesClient := services.NewClient(servicespb.NewServicesServiceClient(dataServiceConn), cfg.Installation) + gameStateClient := gamestate.NewClient(gspb.NewGameStateServiceClient(dataServiceConn), cfg.Installation) + + checkersWorker := worker.New(temporalClient, "checkers", worker.Options{ + DisableRegistrationAliasing: true, + }) + + // Activities. + checkersWorker.RegisterActivityWithOptions( + checkers.NewCheckActivity().ActivityDefinition, + activity.RegisterOptions{Name: checkers.CheckActivityName}, + ) + + checkersWorker.RegisterActivityWithOptions( + checkers.NewFetchDataActivity( + teamsClient, + servicesClient, + gameStateClient, + ).ActivityDefinition, + activity.RegisterOptions{Name: checkers.FetchDataActivityName}, + ) + + checkersWorker.RegisterActivityWithOptions( + checkers.NewGetActivity().ActivityDefinition, + activity.RegisterOptions{Name: checkers.GetActivityName}, + ) + + checkersWorker.RegisterActivityWithOptions( + checkers.NewPickGetFlagActivity(checkersController).ActivityDefinition, + activity.RegisterOptions{Name: checkers.PickGetFlagActivityName}, + ) + + checkersWorker.RegisterActivityWithOptions( + checkers.NewPrepareRoundActivity(checkersController, gameStateClient).ActivityDefinition, + activity.RegisterOptions{Name: checkers.PrepareRoundActivityName}, + ) + + checkersWorker.RegisterActivityWithOptions( + checkers.NewPutActivity().ActivityDefinition, + activity.RegisterOptions{Name: checkers.PutActivityName}, + ) + + checkersWorker.RegisterActivityWithOptions( + checkers.NewSaveRoundDataActivity(checkersController).ActivityDefinition, + activity.RegisterOptions{Name: checkers.SaveRoundDataActivityName}, + ) + + checkersWorker.RegisterActivityWithOptions( + checkers.NewSaveVerdictActivity(checkersController).ActivityDefinition, + activity.RegisterOptions{Name: checkers.SaveVerdictActivityName}, + ) + + checkersWorker.RegisterActivityWithOptions( + checkers.NewGetLastExecutionActivity(checkersController).ActivityDefinition, + activity.RegisterOptions{Name: checkers.GetLastExecutionActivityName}, + ) + // End of activities. + + // Workflows. + checkersWorker.RegisterWorkflowWithOptions( + checkers.CheckWorkflowDefinition, + workflow.RegisterOptions{Name: checkers.CheckWorkflowName}, + ) + + checkersWorker.RegisterWorkflowWithOptions( + checkers.GetWorkflowDefinition, + workflow.RegisterOptions{Name: checkers.GetWorkflowName}, + ) + + checkersWorker.RegisterWorkflowWithOptions( + checkers.RoundWorkflowDefinition, + workflow.RegisterOptions{Name: checkers.RoundWorkflowName}, + ) + // End of workflows. + + if cfg.MetricsAddress != "" { + go metrics.RunServer(runCtx, shutdownCtx, cfg.MetricsAddress) + } + + go func() { + <-runCtx.Done() + checkersWorker.Stop() + }() + if err := checkersWorker.Run(nil); err != nil { + return fmt.Errorf("running worker: %w", err) + } + + return nil +} diff --git a/cmd/checkers/main.go b/cmd/checkers/main.go new file mode 100644 index 0000000..1d969d7 --- /dev/null +++ b/cmd/checkers/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/cmd/checkers/impl" + "github.com/c4t-but-s4d/fastad/internal/checkers" + "github.com/c4t-but-s4d/fastad/pkg/baseconfig" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/stop" +) + +func main() { + defer logging.Init().Close() + + cfg := baseconfig.MustSetupAll(&checkers.Config{}, baseconfig.WithEnvPrefix("FASTAD_CHECKERS")) + + runCtx, shutdownCtx, cancel := stop.SetupCtx() + defer cancel() + + if err := impl.Run(runCtx, shutdownCtx, cfg); err != nil { + zap.L().Fatal("checkers worker run failed", zap.Error(err)) + } +} diff --git a/cmd/dataservice/impl/run.go b/cmd/dataservice/impl/run.go new file mode 100644 index 0000000..dfbb20a --- /dev/null +++ b/cmd/dataservice/impl/run.go @@ -0,0 +1,50 @@ +package impl + +import ( + "context" + "fmt" + + "github.com/c4t-but-s4d/fastad/internal/dataservice" + "github.com/c4t-but-s4d/fastad/internal/dataservice/gamestate" + "github.com/c4t-but-s4d/fastad/internal/dataservice/services" + "github.com/c4t-but-s4d/fastad/internal/dataservice/teams" + "github.com/c4t-but-s4d/fastad/internal/version" + "github.com/c4t-but-s4d/fastad/pkg/grpcext" + "github.com/c4t-but-s4d/fastad/pkg/metrics" + gspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/game_state" + servicespb "github.com/c4t-but-s4d/fastad/pkg/proto/data/services" + teamspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/teams" +) + +func Run(runCtx, shutdownCtx context.Context, cfg *dataservice.Config) error { + db := cfg.Postgres.BunDB() + + versionController := version.NewController(db) + + teamsController := teams.NewController(db, versionController) + teamsService := teams.NewService(teamsController) + + servicesController := services.NewController(db, versionController) + servicesService := services.NewService(servicesController) + + gameStateController := gamestate.NewController(db, versionController) + gameStateService := gamestate.NewService(gameStateController) + + server := grpcext.NewServer( + grpcext.WithServerInstallation(cfg.Installation), + grpcext.WithServerTokenAuth(cfg.IntercomToken), + ) + teamspb.RegisterTeamsServiceServer(server, teamsService) + servicespb.RegisterServicesServiceServer(server, servicesService) + gspb.RegisterGameStateServiceServer(server, gameStateService) + + if cfg.MetricsAddress != "" { + go metrics.RunServer(runCtx, shutdownCtx, cfg.MetricsAddress) + } + + if err := grpcext.RunServer(runCtx, shutdownCtx, server, cfg.ListenAddress); err != nil { + return fmt.Errorf("running server: %w", err) + } + + return nil +} diff --git a/cmd/dataservice/main.go b/cmd/dataservice/main.go new file mode 100644 index 0000000..6274062 --- /dev/null +++ b/cmd/dataservice/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/cmd/dataservice/impl" + "github.com/c4t-but-s4d/fastad/internal/dataservice" + "github.com/c4t-but-s4d/fastad/pkg/baseconfig" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/stop" +) + +func main() { + defer logging.Init().Close() + + cfg := baseconfig.MustSetupAll(&dataservice.Config{}, baseconfig.WithEnvPrefix("FASTAD_DATA_SERVICE")) + + runCtx, shutdownCtx, cancel := stop.SetupCtx() + defer cancel() + + if err := impl.Run(runCtx, shutdownCtx, cfg); err != nil { + zap.L().Fatal("error running server", zap.Error(err)) + } +} diff --git a/cmd/fastad/cmd/context.go b/cmd/fastad/cmd/context.go new file mode 100644 index 0000000..675091b --- /dev/null +++ b/cmd/fastad/cmd/context.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "context" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type Context struct { + Context context.Context //nolint:containedctx // cobra pattern: context passed through command tree + Viper *viper.Viper +} + +type CommandFactory func(cc *Context) *cobra.Command + +func NewContext(v *viper.Viper) *Context { + return &Context{ + Viper: v, + } +} diff --git a/cmd/fastad/cmd/init.go b/cmd/fastad/cmd/init.go new file mode 100644 index 0000000..585912b --- /dev/null +++ b/cmd/fastad/cmd/init.go @@ -0,0 +1,111 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + "github.com/c4t-but-s4d/fastad/internal/gameconfig" +) + +func NewInitCmd(cc *Context) *cobra.Command { + cmd := &cobra.Command{ + Use: "init", + Short: "Initialize game infrastructure (generate compose.yml, .env, etc.)", + RunE: func(_ *cobra.Command, _ []string) error { + root, err := gameconfig.GetFastADRoot() + if err != nil { + return fmt.Errorf("getting fastad root: %w", err) + } + + configPath := cc.Viper.GetString("game_config") + if configPath == "" { + configPath = filepath.Join(root, "fastad.yaml") + } + + zap.S().Infof("reading game config from %s", configPath) + + content, err := os.ReadFile(configPath) + if os.IsNotExist(err) { + examplePath := filepath.Join(root, "fastad.yaml.example") + return fmt.Errorf( + "config file not found: %s\n\nTo get started:\n cp %s %s\n # edit %s with your game settings\n fastad init", + configPath, examplePath, configPath, configPath, + ) + } + if err != nil { + return fmt.Errorf("reading game config: %w", err) + } + + var cfg *gameconfig.GameConfig + if err := yaml.Unmarshal(content, &cfg); err != nil { + return fmt.Errorf("unmarshalling game config: %w", err) + } + + zap.S().Infof("parsed game config: %+v", cfg) + + if err := cfg.Validate(); err != nil { + return fmt.Errorf("validating game config: %w", err) + } + + zap.S().Info("game config validated") + + // Check if already initialized + generatedDir := filepath.Join(root, gameconfig.GeneratedDir) + generatedConfigPath := filepath.Join(generatedDir, gameconfig.GeneratedGameConfig) + if _, err := os.Stat(generatedConfigPath); err == nil && !cc.Viper.GetBool("force") { + return fmt.Errorf( + "game already initialized (config at %s exists). Use --force to reinitialize", + generatedConfigPath, + ) + } + + switch cc.Viper.GetString("preset") { + case "simple": + zap.L().Info("initializing simple preset") + if err := PresetSimple(root, cfg); err != nil { + return fmt.Errorf("initializing simple preset: %w", err) + } + zap.L().Info("simple preset initialized") + case "production": + zap.L().Info("initializing production preset") + if err := PresetProduction(root, cfg); err != nil { + return fmt.Errorf("initializing production preset: %w", err) + } + zap.L().Info("production preset initialized") + default: + return fmt.Errorf("unsupported preset: %s (supported: simple, production)", cc.Viper.GetString("preset")) + } + + // Save generated config + if err := os.MkdirAll(generatedDir, 0o755); err != nil { + return fmt.Errorf("creating generated dir: %w", err) + } + configContent, err := yaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("marshalling game config: %w", err) + } + if err := os.WriteFile(generatedConfigPath, configContent, 0o644); err != nil { + return fmt.Errorf("writing generated game config: %w", err) + } + + zap.L().Info("initialization complete", + zap.String("generated_config", generatedConfigPath), + zap.String("compose_file", filepath.Join(root, "compose.yml")), + ) + zap.L().Info("run 'fastad run' to start the game") + + return nil + }, + } + + cmd.Flags().StringP("game-config", "c", "", "path to game config yaml file (defaults to fastad.yaml in fastad root)") + cmd.Flags().String("preset", "simple", "preset to use ('simple' or 'production')") + cmd.Flags().Bool("force", false, "overwrite existing generated files") + + return cmd +} diff --git a/cmd/fastad/cmd/preset_common.go b/cmd/fastad/cmd/preset_common.go new file mode 100644 index 0000000..68b565b --- /dev/null +++ b/cmd/fastad/cmd/preset_common.go @@ -0,0 +1,169 @@ +package cmd + +import ( + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" +) + +type TemporalPostgresConfig struct { + User string + Password string + Database string + Host string + Port string + TLS string + SkipDBCreate string +} + +func DefaultTemporalPostgresConfig() TemporalPostgresConfig { + return TemporalPostgresConfig{ + Host: "temporal-postgres", + Port: "5432", + User: "temporal", + Password: "temporal", + Database: "temporal", + TLS: "false", + SkipDBCreate: "false", + } +} + +func ParseTemporalDSN(dsn string) (TemporalPostgresConfig, error) { + parsedDSN, err := url.Parse(dsn) + if err != nil { + return TemporalPostgresConfig{}, fmt.Errorf("parsing temporal dsn: %w", err) + } + if parsedDSN.Scheme != "postgres" { + return TemporalPostgresConfig{}, errors.New("temporal dsn scheme must be postgres") + } + + cfg := TemporalPostgresConfig{ + User: parsedDSN.User.Username(), + Host: parsedDSN.Hostname(), + Port: parsedDSN.Port(), + Database: strings.TrimPrefix(parsedDSN.Path, "/"), + TLS: "false", + SkipDBCreate: "true", + } + cfg.Password, _ = parsedDSN.User.Password() + + return cfg, nil +} + +// ComposeFile represents a docker-compose.yml file. +type ComposeFile struct { + Services map[string]any `yaml:"services,omitempty"` + Volumes map[string]any `yaml:"volumes,omitempty"` + Networks map[string]any `yaml:"networks,omitempty"` +} + +// ComposeManipulator provides simple compose file manipulation. +type ComposeManipulator struct { + data *ComposeFile +} + +func LoadCompose(path string) (*ComposeManipulator, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading compose file: %w", err) + } + + var data ComposeFile + if err := yaml.Unmarshal(content, &data); err != nil { + return nil, fmt.Errorf("parsing compose file: %w", err) + } + + if data.Services == nil { + data.Services = make(map[string]any) + } + if data.Volumes == nil { + data.Volumes = make(map[string]any) + } + + return &ComposeManipulator{data: &data}, nil +} + +func (c *ComposeManipulator) Services() map[string]any { + return c.data.Services +} + +func (c *ComposeManipulator) Volumes() map[string]any { + return c.data.Volumes +} + +func (c *ComposeManipulator) RemoveService(name string) { + delete(c.data.Services, name) +} + +func (c *ComposeManipulator) RemoveVolume(name string) { + delete(c.data.Volumes, name) +} + +func (c *ComposeManipulator) getService(name string) map[string]any { + svc, ok := c.data.Services[name] + if !ok { + return nil + } + svcMap, ok := svc.(map[string]any) + if !ok { + return nil + } + return svcMap +} + +func (c *ComposeManipulator) RemoveDependsOn(serviceName string) { + svc := c.getService(serviceName) + if svc == nil { + return + } + delete(svc, "depends_on") +} + +func (c *ComposeManipulator) SetDependsOn(serviceName string, deps map[string]any) { + svc := c.getService(serviceName) + if svc == nil { + return + } + svc["depends_on"] = deps +} + +func (c *ComposeManipulator) SetVolumes(serviceName string, volumes []string) { + svc := c.getService(serviceName) + if svc == nil { + return + } + svc["volumes"] = volumes +} + +func (c *ComposeManipulator) Write(path string) error { + content, err := yaml.Marshal(c.data) + if err != nil { + return fmt.Errorf("marshalling compose file: %w", err) + } + + if err := os.WriteFile(path, content, 0o644); err != nil { + return fmt.Errorf("writing compose file: %w", err) + } + + zap.L().Info("wrote compose file", zap.String("path", path)) + return nil +} + +func WriteEnvFile(path string, content []byte) error { + if err := os.WriteFile(path, content, 0o644); err != nil { + return fmt.Errorf("writing env file: %w", err) + } + + zap.L().Info("wrote .env file", zap.String("path", path)) + return nil +} + +func PresetComposePath(root, preset string) string { + return filepath.Join(root, "docker", "presets", preset, "compose.yml") +} diff --git a/cmd/fastad/cmd/preset_common_test.go b/cmd/fastad/cmd/preset_common_test.go new file mode 100644 index 0000000..c47be52 --- /dev/null +++ b/cmd/fastad/cmd/preset_common_test.go @@ -0,0 +1,230 @@ +package cmd_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/c4t-but-s4d/fastad/cmd/fastad/cmd" +) + +func TestDefaultTemporalPostgresConfig(t *testing.T) { + cfg := cmd.DefaultTemporalPostgresConfig() + + assert.Equal(t, "temporal-postgres", cfg.Host) + assert.Equal(t, "5432", cfg.Port) + assert.Equal(t, "temporal", cfg.User) + assert.Equal(t, "temporal", cfg.Password) + assert.Equal(t, "temporal", cfg.Database) + assert.Equal(t, "false", cfg.TLS) + assert.Equal(t, "false", cfg.SkipDBCreate) +} + +func TestParseTemporalDSN(t *testing.T) { + tests := []struct { + name string + dsn string + want cmd.TemporalPostgresConfig + wantErr string + }{ + { + name: "full dsn", + dsn: "postgres://user:pass@host:5433/mydb", + want: cmd.TemporalPostgresConfig{ + User: "user", + Password: "pass", + Host: "host", + Port: "5433", + Database: "mydb", + TLS: "false", + SkipDBCreate: "true", + }, + }, + { + name: "dsn without password", + dsn: "postgres://user@host:5432/mydb", + want: cmd.TemporalPostgresConfig{ + User: "user", + Password: "", + Host: "host", + Port: "5432", + Database: "mydb", + TLS: "false", + SkipDBCreate: "true", + }, + }, + { + name: "dsn with leading slash in path", + dsn: "postgres://user:pass@host:5432/database", + want: cmd.TemporalPostgresConfig{ + User: "user", + Password: "pass", + Host: "host", + Port: "5432", + Database: "database", + TLS: "false", + SkipDBCreate: "true", + }, + }, + { + name: "invalid scheme", + dsn: "mysql://user:pass@host:3306/db", + wantErr: "scheme must be postgres", + }, + { + name: "invalid url", + dsn: "://invalid", + wantErr: "parsing temporal dsn", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := cmd.ParseTemporalDSN(tt.dsn) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + } else { + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +func TestComposeManipulator(t *testing.T) { + tmpDir := t.TempDir() + composePath := filepath.Join(tmpDir, "compose.yml") + + initialContent := `services: + app: + image: myapp + depends_on: + postgres: + condition: service_healthy + volumes: + - ./data:/data + postgres: + image: postgres:17 + redis: + image: redis:7 +volumes: + app-data: + postgres-data: +` + + err := os.WriteFile(composePath, []byte(initialContent), 0o644) + require.NoError(t, err) + + t.Run("LoadCompose", func(t *testing.T) { + compose, err := cmd.LoadCompose(composePath) + require.NoError(t, err) + assert.Len(t, compose.Services(), 3) + assert.Len(t, compose.Volumes(), 2) + }) + + t.Run("RemoveService", func(t *testing.T) { + compose, err := cmd.LoadCompose(composePath) + require.NoError(t, err) + + compose.RemoveService("postgres") + assert.Len(t, compose.Services(), 2) + _, exists := compose.Services()["postgres"] + assert.False(t, exists) + }) + + t.Run("RemoveVolume", func(t *testing.T) { + compose, err := cmd.LoadCompose(composePath) + require.NoError(t, err) + + compose.RemoveVolume("postgres-data") + assert.Len(t, compose.Volumes(), 1) + _, exists := compose.Volumes()["postgres-data"] + assert.False(t, exists) + }) + + t.Run("RemoveDependsOn", func(t *testing.T) { + compose, err := cmd.LoadCompose(composePath) + require.NoError(t, err) + + compose.RemoveDependsOn("app") + appService, ok := compose.Services()["app"].(map[string]any) + require.True(t, ok, "app service should be a map") + _, hasDeps := appService["depends_on"] + assert.False(t, hasDeps) + }) + + t.Run("SetDependsOn", func(t *testing.T) { + compose, err := cmd.LoadCompose(composePath) + require.NoError(t, err) + + newDeps := map[string]any{ + "redis": map[string]any{"condition": "service_started"}, + } + compose.SetDependsOn("app", newDeps) + + appService, ok := compose.Services()["app"].(map[string]any) + require.True(t, ok, "app service should be a map") + deps, ok := appService["depends_on"].(map[string]any) + require.True(t, ok, "depends_on should be a map") + redisDep, ok := deps["redis"].(map[string]any) + require.True(t, ok, "redis dep should be a map") + assert.Equal(t, "service_started", redisDep["condition"]) + }) + + t.Run("SetVolumes", func(t *testing.T) { + compose, err := cmd.LoadCompose(composePath) + require.NoError(t, err) + + compose.SetVolumes("app", []string{"/new/path:/container/path"}) + + appService, ok := compose.Services()["app"].(map[string]any) + require.True(t, ok, "app service should be a map") + volumes, ok := appService["volumes"].([]string) + require.True(t, ok, "volumes should be a string slice") + require.Len(t, volumes, 1) + assert.Equal(t, "/new/path:/container/path", volumes[0]) + }) + + t.Run("Write", func(t *testing.T) { + compose, err := cmd.LoadCompose(composePath) + require.NoError(t, err) + + compose.RemoveService("redis") + + outputPath := filepath.Join(tmpDir, "output.yml") + err = compose.Write(outputPath) + require.NoError(t, err) + + reloaded, err := cmd.LoadCompose(outputPath) + require.NoError(t, err) + assert.Len(t, reloaded.Services(), 2) + }) + + t.Run("LoadCompose_FileNotFound", func(t *testing.T) { + _, err := cmd.LoadCompose("/nonexistent/path/compose.yml") + require.Error(t, err) + assert.Contains(t, err.Error(), "reading compose file") + }) +} + +func TestWriteEnvFile(t *testing.T) { + tmpDir := t.TempDir() + envPath := filepath.Join(tmpDir, ".env") + + content := []byte("KEY=value\nANOTHER=test\n") + err := cmd.WriteEnvFile(envPath, content) + require.NoError(t, err) + + read, err := os.ReadFile(envPath) + require.NoError(t, err) + assert.Equal(t, content, read) +} + +func TestPresetComposePath(t *testing.T) { + path := cmd.PresetComposePath("/root/project", "simple") + assert.Equal(t, "/root/project/docker/presets/simple/compose.yml", path) +} diff --git a/cmd/fastad/cmd/preset_production.go b/cmd/fastad/cmd/preset_production.go new file mode 100644 index 0000000..7e1aff5 --- /dev/null +++ b/cmd/fastad/cmd/preset_production.go @@ -0,0 +1,117 @@ +package cmd + +import ( + "bytes" + "fmt" + "path/filepath" + "text/template" + + "github.com/c4t-but-s4d/fastad/internal/gameconfig" +) + +type productionEnvTemplateContext struct { + LogLevel string + PublicPort int + IntercomToken string + CheckersDir string + + DatabaseDSN string + Temporal TemporalPostgresConfig +} + +const productionEnvTemplateData = `# FastAD production preset environment file +# This file was generated by FastAD setup +# Do not edit this file manually + +# services +FASTAD_LOG_LEVEL={{ .LogLevel }} +FASTAD_POSTGRES_DSN={{ .DatabaseDSN }} +FASTAD_INTERCOM_TOKEN={{ .IntercomToken }} + +# checkers +CHECKERS_DIR={{ .CheckersDir }} + +# temporal +TEMPORAL_POSTGRES_USER={{ .Temporal.User }} +TEMPORAL_POSTGRES_PASSWORD={{ .Temporal.Password }} +TEMPORAL_POSTGRES_DATABASE={{ .Temporal.Database }} +TEMPORAL_POSTGRES_HOST={{ .Temporal.Host }} +TEMPORAL_POSTGRES_PORT={{ .Temporal.Port }} +TEMPORAL_POSTGRES_TLS={{ .Temporal.TLS }} +TEMPORAL_SKIP_DB_CREATE={{ .Temporal.SkipDBCreate }} + +# caddy +CADDY_PORT={{ .PublicPort }} +` + +var productionEnvTemplate = template.Must(template.New("productionEnv").Parse(productionEnvTemplateData)) + +func PresetProduction(root string, config *gameconfig.GameConfig) error { + envContext := productionEnvTemplateContext{ + LogLevel: config.FastAD.LogLevel, + PublicPort: config.FastAD.ListenPort, + IntercomToken: config.FastAD.IntercomToken, + } + + checkersPath := config.Game.CheckersBasePath + if filepath.IsAbs(checkersPath) { + relPath, err := filepath.Rel(root, checkersPath) + if err == nil { + checkersPath = relPath + } + } + envContext.CheckersDir = checkersPath + + startPostgres := true + if config.Database != nil && config.Database.ExternalDSN != "" { + envContext.DatabaseDSN = config.Database.ExternalDSN + startPostgres = false + } else { + envContext.DatabaseDSN = "postgres://fastad:fastad@postgres:5432/fastad?sslmode=disable" + } + + startTemporalPostgres := true + if config.Database != nil && config.Database.TemporalExternalDSN != "" { + var err error + envContext.Temporal, err = ParseTemporalDSN(config.Database.TemporalExternalDSN) + if err != nil { + return fmt.Errorf("parsing temporal external dsn: %w", err) + } + startTemporalPostgres = false + } else { + envContext.Temporal = DefaultTemporalPostgresConfig() + } + + var envContent bytes.Buffer + if err := productionEnvTemplate.Execute(&envContent, envContext); err != nil { + return fmt.Errorf("executing env template: %w", err) + } + + envPath := filepath.Join(root, ".env") + if err := WriteEnvFile(envPath, envContent.Bytes()); err != nil { + return err + } + + compose, err := LoadCompose(PresetComposePath(root, "production")) + if err != nil { + return err + } + + if !startPostgres { + compose.RemoveService("postgres") + compose.RemoveDependsOn("migrator") + } + + if !startTemporalPostgres { + compose.RemoveService("temporal-postgres") + compose.RemoveDependsOn("temporal-admin-tools") + } + + config.Game.CheckersBasePath = "/checkers-scripts" + for _, service := range config.Services { + service.Checker.Path = filepath.Join(config.Game.CheckersBasePath, service.Checker.Path) + } + + destCompose := filepath.Join(root, "compose.yml") + return compose.Write(destCompose) +} diff --git a/cmd/fastad/cmd/preset_simple.go b/cmd/fastad/cmd/preset_simple.go new file mode 100644 index 0000000..447c84c --- /dev/null +++ b/cmd/fastad/cmd/preset_simple.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "bytes" + "fmt" + "path/filepath" + "text/template" + + "github.com/c4t-but-s4d/fastad/internal/gameconfig" +) + +type simpleEnvTemplateContext struct { + LogLevel string + PublicPort int + IntercomToken string + + DatabaseDSN string + Temporal TemporalPostgresConfig +} + +const simpleEnvTemplateData = `# FastAD simple preset environment file +# This file was generated by FastAD setup +# Do not edit this file manually + +# allinone +FASTAD_LOG_LEVEL={{ .LogLevel }} +FASTAD_POSTGRES_DSN={{ .DatabaseDSN }} +FASTAD_INTERCOM_TOKEN={{ .IntercomToken }} + +# temporal +TEMPORAL_POSTGRES_USER={{ .Temporal.User }} +TEMPORAL_POSTGRES_PASSWORD={{ .Temporal.Password }} +TEMPORAL_POSTGRES_DATABASE={{ .Temporal.Database }} +TEMPORAL_POSTGRES_HOST={{ .Temporal.Host }} +TEMPORAL_POSTGRES_PORT={{ .Temporal.Port }} +TEMPORAL_POSTGRES_TLS={{ .Temporal.TLS }} +TEMPORAL_SKIP_DB_CREATE={{ .Temporal.SkipDBCreate }} + +# caddy +CADDY_PORT={{ .PublicPort }} +` + +var simpleEnvTemplate = template.Must(template.New("simpleEnv").Parse(simpleEnvTemplateData)) + +func PresetSimple(root string, config *gameconfig.GameConfig) error { + envContext := simpleEnvTemplateContext{ + LogLevel: config.FastAD.LogLevel, + PublicPort: config.FastAD.ListenPort, + IntercomToken: config.FastAD.IntercomToken, + } + + startPostgres := true + if config.Database != nil && config.Database.ExternalDSN != "" { + envContext.DatabaseDSN = config.Database.ExternalDSN + startPostgres = false + } else { + envContext.DatabaseDSN = "postgres://fastad:fastad@postgres:5432/fastad?sslmode=disable" + } + + startTemporalPostgres := true + if config.Database != nil && config.Database.TemporalExternalDSN != "" { + var err error + envContext.Temporal, err = ParseTemporalDSN(config.Database.TemporalExternalDSN) + if err != nil { + return fmt.Errorf("parsing temporal external dsn: %w", err) + } + startTemporalPostgres = false + } else { + envContext.Temporal = DefaultTemporalPostgresConfig() + } + + var envContent bytes.Buffer + if err := simpleEnvTemplate.Execute(&envContent, envContext); err != nil { + return fmt.Errorf("executing env template: %w", err) + } + + envPath := filepath.Join(root, ".env") + if err := WriteEnvFile(envPath, envContent.Bytes()); err != nil { + return err + } + + compose, err := LoadCompose(PresetComposePath(root, "simple")) + if err != nil { + return err + } + + if !startPostgres { + compose.RemoveService("postgres") + compose.RemoveVolume("fastad-db") + compose.SetDependsOn("app", map[string]any{ + "temporal": map[string]any{ + "condition": "service_healthy", + }, + }) + } + + if !startTemporalPostgres { + compose.RemoveService("temporal-postgres") + compose.RemoveVolume("temporal-db") + compose.RemoveDependsOn("temporal-admin-tools") + } + + checkersPath := config.Game.CheckersBasePath + if !filepath.IsAbs(checkersPath) { + checkersPath = filepath.Join(root, checkersPath) + } + + compose.SetVolumes("app", []string{ + fmt.Sprintf("%s:/checkers", checkersPath), + }) + + config.Game.CheckersBasePath = "/checkers" + for _, service := range config.Services { + service.Checker.Path = filepath.Join(config.Game.CheckersBasePath, service.Checker.Path) + } + + destCompose := filepath.Join(root, "compose.yml") + return compose.Write(destCompose) +} diff --git a/cmd/fastad/cmd/reset.go b/cmd/fastad/cmd/reset.go new file mode 100644 index 0000000..0e81d82 --- /dev/null +++ b/cmd/fastad/cmd/reset.go @@ -0,0 +1,55 @@ +package cmd + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/spf13/cobra" + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/internal/gameconfig" +) + +func NewResetCmd(_ *Context) *cobra.Command { + cmd := &cobra.Command{ + Use: "reset", + Short: "Reset the game (stops containers, removes volumes and data)", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() + root, err := gameconfig.GetFastADRoot() + if err != nil { + return fmt.Errorf("getting fastad root: %w", err) + } + + composePath := filepath.Join(root, "compose.yml") + if _, err := os.Stat(composePath); os.IsNotExist(err) { + zap.L().Info("compose.yml not found, nothing to reset") + return nil + } + + zap.L().Info("stopping and removing containers") + stopCmd := exec.CommandContext(ctx, "docker", "compose", "-f", composePath, "down", "-v", "--remove-orphans") + stopCmd.Dir = root + stopCmd.Stdout = os.Stdout + stopCmd.Stderr = os.Stderr + if err := stopCmd.Run(); err != nil { + return fmt.Errorf("stopping containers: %w", err) + } + + dataDir := filepath.Join(root, "data") + if _, err := os.Stat(dataDir); err == nil { + zap.L().Info("removing data directory", zap.String("path", dataDir)) + if err := os.RemoveAll(dataDir); err != nil { + zap.L().Warn("failed to remove data directory", zap.Error(err)) + } + } + + zap.L().Info("reset complete") + return nil + }, + } + + return cmd +} diff --git a/cmd/fastad/cmd/root.go b/cmd/fastad/cmd/root.go new file mode 100644 index 0000000..e444462 --- /dev/null +++ b/cmd/fastad/cmd/root.go @@ -0,0 +1,24 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + "github.com/c4t-but-s4d/fastad/pkg/viperext" +) + +func NewRootCmd(cc *Context) *cobra.Command { + rootCmd := &cobra.Command{ + Use: "fastad", + Short: "FastAD - Blazing fast Attack & Defence CTF platform", + SilenceUsage: true, + SilenceErrors: true, + TraverseChildren: true, + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + viperext.RunBindCommandFlags(cc.Viper, cmd) + cc.Context = cmd.Context() + return nil + }, + } + + return rootCmd +} diff --git a/cmd/fastad/cmd/run.go b/cmd/fastad/cmd/run.go new file mode 100644 index 0000000..caa412e --- /dev/null +++ b/cmd/fastad/cmd/run.go @@ -0,0 +1,144 @@ +package cmd + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/samber/lo" + "github.com/spf13/cobra" + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/internal/gameconfig" + "github.com/c4t-but-s4d/fastad/pkg/apiwait" + "github.com/c4t-but-s4d/fastad/pkg/clients/gamestate" + "github.com/c4t-but-s4d/fastad/pkg/clients/services" + "github.com/c4t-but-s4d/fastad/pkg/clients/teams" + "github.com/c4t-but-s4d/fastad/pkg/grpcext" + gspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/game_state" + servicespb "github.com/c4t-but-s4d/fastad/pkg/proto/data/services" + teamspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/teams" +) + +func NewRunCmd(cc *Context) *cobra.Command { + cmd := &cobra.Command{ + Use: "run", + Short: "Start the game (runs docker compose and initializes game state)", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() + root, err := gameconfig.GetFastADRoot() + if err != nil { + return fmt.Errorf("getting fastad root: %w", err) + } + + // Read the generated config (created by 'fastad init') + cfg, err := gameconfig.ReadGeneratedConfig() + if err != nil { + return fmt.Errorf("reading generated config: %w (did you run 'fastad init' first?)", err) + } + + composePath := filepath.Join(root, "compose.yml") + if _, err := os.Stat(composePath); os.IsNotExist(err) { + return fmt.Errorf("compose.yml not found at %s (did you run 'fastad init' first?)", composePath) + } + + if !cc.Viper.GetBool("only_init") { + // Stop existing services first + zap.L().Info("stopping existing services") + stopCmd := exec.CommandContext(ctx, "docker", "compose", "-f", composePath, "down", "-v", "--remove-orphans") + stopCmd.Dir = root + stopCmd.Stdout = os.Stdout + stopCmd.Stderr = os.Stderr + if err := stopCmd.Run(); err != nil { + return fmt.Errorf("stopping existing services: %w", err) + } + + // Start services + args := []string{"compose", "-f", composePath, "up", "-d"} + if !cc.Viper.GetBool("no_build") { + args = append(args, "--build") + } + + dockerCmd := exec.CommandContext(ctx, "docker", args...) + dockerCmd.Dir = root + dockerCmd.Stdout = os.Stdout + dockerCmd.Stderr = os.Stderr + + zap.L().Info("starting services", zap.String("command", dockerCmd.String())) + if err := dockerCmd.Run(); err != nil { + return fmt.Errorf("running docker compose: %w", err) + } + } else { + zap.L().Info("skipping starting the services (--only-init)") + } + + // Wait for services to be ready + zap.L().Info("waiting for services to start") + httpAddr := fmt.Sprintf("127.0.0.1:%d", cfg.FastAD.ListenPort) + if err := apiwait.HTTP(ctx, httpAddr); err != nil { + return fmt.Errorf("waiting for API service: %w", err) + } + + // Initialize game state + apiAddress := fmt.Sprintf("127.0.0.1:%d", cfg.FastAD.ListenPort) + zap.L().Info("initializing game", zap.String("api_address", apiAddress)) + + apiConn, err := grpcext.Dial( + apiAddress, + "fastad-setup", + grpcext.AuthDialOptions(cfg.FastAD.IntercomToken)..., + ) + if err != nil { + return fmt.Errorf("dialing data service: %w", err) + } + defer apiConn.Close() + + teamsClient := teams.NewClient(teamspb.NewTeamsServiceClient(apiConn), "fastad-cli") + servicesClient := services.NewClient(servicespb.NewServicesServiceClient(apiConn), "fastad-cli") + gameStateClient := gamestate.NewClient(gspb.NewGameStateServiceClient(apiConn), "fastad-cli") + + teamsToCreate := lo.Map(cfg.Teams, func(t *gameconfig.Team, _ int) *teamspb.Team { + return t.ToProto() + }) + + servicesToCreate := lo.Map(cfg.Services, func(s *gameconfig.Service, _ int) *servicespb.Service { + return s.ToProto() + }) + + createdTeams, err := teamsClient.CreateBatch(ctx, teamsToCreate) + if err != nil { + return fmt.Errorf("creating teams: %w", err) + } + zap.S().Infof("created teams: %v", createdTeams) + + createdServices, err := servicesClient.CreateBatch(ctx, servicesToCreate) + if err != nil { + return fmt.Errorf("creating services: %w", err) + } + zap.S().Infof("created services: %v", createdServices) + + // Try to get existing game state first + existingGameState, err := gameStateClient.Get(ctx) + if err == nil && existingGameState != nil { + zap.S().Infof("game state already exists, skipping creation: %+v", existingGameState) + } else { + // Create new game state + createdGameState, err := gameStateClient.Create(ctx, cfg.Game.ToCreateRequestProto()) + if err != nil { + return fmt.Errorf("creating game state: %w", err) + } + zap.S().Infof("created game state: %+v", createdGameState) + } + + zap.L().Info("game started successfully") + + return nil + }, + } + + cmd.Flags().Bool("no-build", false, "skip building images (use --build by default)") + cmd.Flags().Bool("only-init", false, "only initialize the game state (skip starting containers)") + + return cmd +} diff --git a/cmd/fastad/cmd/tokens.go b/cmd/fastad/cmd/tokens.go new file mode 100644 index 0000000..2c2a77f --- /dev/null +++ b/cmd/fastad/cmd/tokens.go @@ -0,0 +1,78 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/c4t-but-s4d/fastad/internal/gameconfig" + "github.com/c4t-but-s4d/fastad/pkg/clients/teams" + "github.com/c4t-but-s4d/fastad/pkg/grpcext" + teamspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/teams" +) + +func NewTokensCmd(cc *Context) *cobra.Command { + cmd := &cobra.Command{ + Use: "tokens", + Short: "Print team tokens", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() + cfg, err := gameconfig.ReadGeneratedConfig() + if err != nil { + return fmt.Errorf("reading generated config: %w", err) + } + + apiAddress := fmt.Sprintf("127.0.0.1:%d", cfg.FastAD.ListenPort) + apiConn, err := grpcext.Dial( + apiAddress, + "fastad-setup", + grpcext.AuthDialOptions(cfg.FastAD.IntercomToken)..., + ) + if err != nil { + return fmt.Errorf("dialing data service: %w", err) + } + defer apiConn.Close() + + teamsClient := teams.NewClient(teamspb.NewTeamsServiceClient(apiConn), "fastad-cli") + + teamsList, err := teamsClient.List(ctx) + if err != nil { + return fmt.Errorf("listing teams: %w", err) + } + + switch cc.Viper.GetString("format") { + case "json": + type teamToken struct { + ID int64 `json:"id"` + Name string `json:"name"` + Token string `json:"token"` + } + data := make([]teamToken, 0, len(teamsList)) + for _, team := range teamsList { + data = append(data, teamToken{ + ID: team.GetId(), + Name: team.GetName(), + Token: team.GetToken(), + }) + } + if err := json.NewEncoder(os.Stdout).Encode(data); err != nil { + return fmt.Errorf("encoding JSON: %w", err) + } + case "text": + for _, team := range teamsList { + fmt.Printf("%s:%s\n", team.GetName(), team.GetToken()) + } + default: + return fmt.Errorf("unsupported format: %s", cc.Viper.GetString("format")) + } + + return nil + }, + } + + cmd.Flags().StringP("format", "f", "text", "output format (json, text)") + + return cmd +} diff --git a/cmd/fastad/main.go b/cmd/fastad/main.go new file mode 100644 index 0000000..5c90eb2 --- /dev/null +++ b/cmd/fastad/main.go @@ -0,0 +1,40 @@ +package main + +import ( + "context" + "os/signal" + "syscall" + + "github.com/spf13/cobra" + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/cmd/fastad/cmd" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/viperext" +) + +func main() { + cobra.EnableTraverseRunHooks = true + + defer logging.Init().Close() + + v, err := viperext.NewViper("FASTAD") + if err != nil { + zap.L().Fatal("creating viper", zap.Error(err)) + } + + cc := cmd.NewContext(v) + + rootCmd := cmd.NewRootCmd(cc) + rootCmd.AddCommand(cmd.NewInitCmd(cc)) + rootCmd.AddCommand(cmd.NewRunCmd(cc)) + rootCmd.AddCommand(cmd.NewResetCmd(cc)) + rootCmd.AddCommand(cmd.NewTokensCmd(cc)) + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer cancel() + + if err := rootCmd.ExecuteContext(ctx); err != nil { + zap.L().Fatal("running app", zap.Error(err)) + } +} diff --git a/cmd/migrator/main.go b/cmd/migrator/main.go new file mode 100644 index 0000000..7da5430 --- /dev/null +++ b/cmd/migrator/main.go @@ -0,0 +1,215 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/migrate" + "github.com/urfave/cli/v2" + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/cmd/migrator/migrations" + "github.com/c4t-but-s4d/fastad/pkg/baseconfig" + "github.com/c4t-but-s4d/fastad/pkg/config" + "github.com/c4t-but-s4d/fastad/pkg/logging" +) + +const ( + maxRetries = 10 + initialBackoff = 1 * time.Second + maxBackoff = 30 * time.Second +) + +func connectWithRetry(ctx context.Context, pgCfg *config.Postgres) (*bun.DB, error) { + var db *bun.DB + backoff := initialBackoff + + for attempt := 1; attempt <= maxRetries; attempt++ { + db = pgCfg.BunDB() + + if err := db.PingContext(ctx); err != nil { + zap.L().Warn( + "failed to connect to postgres, retrying", + zap.Int("attempt", attempt), + zap.Int("max_retries", maxRetries), + zap.Duration("backoff", backoff), + zap.Error(err), + ) + + if err := db.Close(); err != nil { + zap.L().Warn("failed to close db connection", zap.Error(err)) + } + + if attempt == maxRetries { + return nil, fmt.Errorf("failed to connect to postgres after %d attempts: %w", maxRetries, err) + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(backoff): + } + + backoff *= 2 + if backoff > maxBackoff { + backoff = maxBackoff + } + continue + } + + zap.L().Info("connected to postgres", zap.Int("attempt", attempt)) + return db, nil + } + + return nil, fmt.Errorf("failed to connect to postgres after %d attempts", maxRetries) +} + +type Config struct { + Postgres config.Postgres `mapstructure:"postgres"` +} + +func main() { + defer logging.Init().Close() + + app := cli.NewApp() + + var cfg *Config + var migrator *migrate.Migrator + app.Before = func(c *cli.Context) error { + var err error + if cfg, err = baseconfig.SetupAll(&Config{}, baseconfig.WithEnvPrefix("FASTAD_MIGRATOR")); err != nil { + return fmt.Errorf("setting up config: %w", err) + } + + db, err := connectWithRetry(c.Context, &cfg.Postgres) + if err != nil { + return fmt.Errorf("connecting to postgres: %w", err) + } + + migrator = migrate.NewMigrator(db, migrations.Migrations) + return nil + } + + app.Commands = []*cli.Command{ + { + Name: "init", + Usage: "create migration tables", + Action: func(c *cli.Context) error { + return migrator.Init(c.Context) + }, + }, + { + Name: "migrate", + Usage: "migrate database", + Action: func(c *cli.Context) error { + group, err := migrator.Migrate(c.Context) + if err != nil { + return fmt.Errorf("migrating: %w", err) + } + + if group.ID == 0 { + zap.L().Info("there are no new migrations to run") + return nil + } + + zap.S().Infof("migrated to %s", group) + return nil + }, + }, + { + Name: "rollback", + Usage: "rollback the last migration group", + Action: func(c *cli.Context) error { + group, err := migrator.Rollback(c.Context) + if err != nil { + return fmt.Errorf("rolling back: %w", err) + } + + if group.ID == 0 { + zap.L().Info("there are no groups to roll back") + return nil + } + + zap.S().Infof("rolled back %s", group) + return nil + }, + }, + { + Name: "create_go", + Usage: "create Go migration", + Action: func(c *cli.Context) error { + name := strings.Join(c.Args().Slice(), "_") + mf, err := migrator.CreateGoMigration(c.Context, name) + if err != nil { + return fmt.Errorf("creating Go migration: %w", err) + } + zap.S().Infof("created migration %s (%s)", mf.Name, mf.Path) + + return nil + }, + }, + { + Name: "create_sql", + Usage: "create up and down SQL migrations", + Action: func(c *cli.Context) error { + name := strings.Join(c.Args().Slice(), "_") + files, err := migrator.CreateSQLMigrations(c.Context, name) + if err != nil { + return fmt.Errorf("creating SQL migrations: %w", err) + } + + for _, mf := range files { + zap.S().Infof("created migration %s (%s)", mf.Name, mf.Path) + } + + return nil + }, + }, + { + Name: "status", + Usage: "print migrations status", + Action: func(c *cli.Context) error { + ms, err := migrator.MigrationsWithStatus(c.Context) + if err != nil { + return fmt.Errorf("checking migrations status: %w", err) + } + zap.S().Infof("migrations: %s", ms) + zap.S().Infof("unapplied migrations: %s", ms.Unapplied()) + zap.S().Infof("last migration group: %s", ms.LastGroup()) + + return nil + }, + }, + { + Name: "mark_applied", + Usage: "mark migrations as applied without actually running them", + Action: func(c *cli.Context) error { + group, err := migrator.Migrate(c.Context, migrate.WithNopMigration()) + if err != nil { + return fmt.Errorf("migrating: %w", err) + } + + if group.ID == 0 { + zap.L().Info("there are no new migrations to mark as applied") + return nil + } + + zap.S().Infof("marked as applied %s", group) + return nil + }, + }, + } + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + if err := app.RunContext(ctx, os.Args); err != nil { + zap.L().Fatal("error running app", zap.Error(err)) + } +} diff --git a/cmd/migrator/migrations/20241223100353_data_service_models.go b/cmd/migrator/migrations/20241223100353_data_service_models.go new file mode 100644 index 0000000..0480397 --- /dev/null +++ b/cmd/migrator/migrations/20241223100353_data_service_models.go @@ -0,0 +1,47 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/uptrace/bun" + + "github.com/c4t-but-s4d/fastad/internal/models" +) + +//nolint:gochecknoinits // Migrations should be initialized in init functions. +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + if _, err := db.NewCreateTable(). + Model((*models.Version)(nil)). + IfNotExists(). + Exec(ctx); err != nil { + return fmt.Errorf("creating versions: %w", err) + } + + if _, err := db.NewCreateTable(). + Model((*models.Team)(nil)). + IfNotExists(). + Exec(ctx); err != nil { + return fmt.Errorf("creating teams: %w", err) + } + + if _, err := db.NewCreateTable(). + Model((*models.Service)(nil)). + IfNotExists(). + Exec(ctx); err != nil { + return fmt.Errorf("creating services: %w", err) + } + + if _, err := db.NewCreateTable(). + Model((*models.GameState)(nil)). + IfNotExists(). + Exec(ctx); err != nil { + return fmt.Errorf("creating game_states: %w", err) + } + + return nil + }, func(context.Context, *bun.DB) error { + return nil + }) +} diff --git a/cmd/migrator/migrations/20241223100559_checkers_models.go b/cmd/migrator/migrations/20241223100559_checkers_models.go new file mode 100644 index 0000000..89587f7 --- /dev/null +++ b/cmd/migrator/migrations/20241223100559_checkers_models.go @@ -0,0 +1,46 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/uptrace/bun" + + "github.com/c4t-but-s4d/fastad/internal/models" +) + +//nolint:gochecknoinits // Migrations should be initialized in init functions. +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + if _, err := db. + NewCreateTable(). + Model((*models.CheckerExecution)(nil)). + IfNotExists(). + WithForeignKeys(). + Exec(ctx); err != nil { + return fmt.Errorf("creating checkers_executions: %w", err) + } + + if _, err := db. + NewCreateTable(). + Model((*models.Flag)(nil)). + IfNotExists(). + WithForeignKeys(). + Exec(ctx); err != nil { + return fmt.Errorf("creating flags: %w", err) + } + + if _, err := db. + NewCreateTable(). + Model((*models.AttackDataSnapshot)(nil)). + IfNotExists(). + WithForeignKeys(). + Exec(ctx); err != nil { + return fmt.Errorf("creating attack_data_snapshots: %w", err) + } + + return nil + }, func(context.Context, *bun.DB) error { + return nil + }) +} diff --git a/cmd/migrator/migrations/20241223101112_receiver_models.go b/cmd/migrator/migrations/20241223101112_receiver_models.go new file mode 100644 index 0000000..0ebad10 --- /dev/null +++ b/cmd/migrator/migrations/20241223101112_receiver_models.go @@ -0,0 +1,28 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/uptrace/bun" + + "github.com/c4t-but-s4d/fastad/internal/models" +) + +//nolint:gochecknoinits // Migrations should be initialized in init functions. +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + if _, err := db. + NewCreateTable(). + Model((*models.Attack)(nil)). + IfNotExists(). + WithForeignKeys(). + Exec(ctx); err != nil { + return fmt.Errorf("creating attacks: %w", err) + } + + return nil + }, func(context.Context, *bun.DB) error { + return nil + }) +} diff --git a/cmd/migrator/migrations/20241223103441_scheduler_models.go b/cmd/migrator/migrations/20241223103441_scheduler_models.go new file mode 100644 index 0000000..8dbf5de --- /dev/null +++ b/cmd/migrator/migrations/20241223103441_scheduler_models.go @@ -0,0 +1,26 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/uptrace/bun" + + "github.com/c4t-but-s4d/fastad/internal/models" +) + +//nolint:gochecknoinits // Migrations should be initialized in init functions. +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + if _, err := db.NewCreateTable(). + Model((*models.SchedulerState)(nil)). + IfNotExists(). + Exec(ctx); err != nil { + return fmt.Errorf("creating scheduler_states: %w", err) + } + + return nil + }, func(context.Context, *bun.DB) error { + return nil + }) +} diff --git a/cmd/migrator/migrations/20241223124709_data_service_aux.up.sql b/cmd/migrator/migrations/20241223124709_data_service_aux.up.sql new file mode 100644 index 0000000..b957d08 --- /dev/null +++ b/cmd/migrator/migrations/20241223124709_data_service_aux.up.sql @@ -0,0 +1,6 @@ +SET statement_timeout = 0; + +--bun:split + +ALTER TABLE game_state + ADD CONSTRAINT hardness_non_zero CHECK (hardness > 0); diff --git a/cmd/migrator/migrations/20241224083434_checkers_aux.up.sql b/cmd/migrator/migrations/20241224083434_checkers_aux.up.sql new file mode 100644 index 0000000..4542a6d --- /dev/null +++ b/cmd/migrator/migrations/20241224083434_checkers_aux.up.sql @@ -0,0 +1,16 @@ +SET statement_timeout = 0; + +--bun:split + +-- For faster flag picking for GET. +CREATE INDEX IF NOT EXISTS idx_flags_team_service_round ON flags (team_id, service_id, round); + +--bun:split + +--For faster slac processing +CREATE INDEX IF NOT EXISTS idx_checker_executions_created_at ON checker_executions (created_at); + +--bun:split + +--For faster attack_data handle +CREATE INDEX IF NOT EXISTS idx_attack_data_snapshots ON attack_data_snapshots (created_at DESC); diff --git a/cmd/migrator/migrations/20241224120831_scoreboard_models.go b/cmd/migrator/migrations/20241224120831_scoreboard_models.go new file mode 100644 index 0000000..8440bb2 --- /dev/null +++ b/cmd/migrator/migrations/20241224120831_scoreboard_models.go @@ -0,0 +1,27 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/uptrace/bun" + + "github.com/c4t-but-s4d/fastad/internal/models" +) + +//nolint:gochecknoinits // Migrations should be initialized in init functions. +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + if _, err := db.NewCreateTable(). + Model((*models.SlacProcessedItem)(nil)). + IfNotExists(). + // Don't create foreign keys here. + Exec(ctx); err != nil { + return fmt.Errorf("creating processor_states: %w", err) + } + + return nil + }, func(context.Context, *bun.DB) error { + return nil + }) +} diff --git a/cmd/migrator/migrations/main.go b/cmd/migrator/migrations/main.go new file mode 100644 index 0000000..c7755e6 --- /dev/null +++ b/cmd/migrator/migrations/main.go @@ -0,0 +1,19 @@ +package migrations + +import ( + "embed" + + "github.com/uptrace/bun/migrate" +) + +var Migrations = migrate.NewMigrations() + +//go:embed *.sql +var sqlMigrations embed.FS + +//nolint:gochecknoinits // Migrations should be initialized in init functions. +func init() { + if err := Migrations.Discover(sqlMigrations); err != nil { + panic(err) + } +} diff --git a/cmd/receiver/impl/run.go b/cmd/receiver/impl/run.go new file mode 100644 index 0000000..e556484 --- /dev/null +++ b/cmd/receiver/impl/run.go @@ -0,0 +1,97 @@ +package impl + +import ( + "context" + "fmt" + + "golang.org/x/sync/errgroup" + + "github.com/c4t-but-s4d/fastad/internal/centutil" + "github.com/c4t-but-s4d/fastad/internal/receiver" + "github.com/c4t-but-s4d/fastad/pkg/clients/gamestate" + "github.com/c4t-but-s4d/fastad/pkg/clients/services" + "github.com/c4t-but-s4d/fastad/pkg/clients/teams" + "github.com/c4t-but-s4d/fastad/pkg/grpcext" + "github.com/c4t-but-s4d/fastad/pkg/metrics" + gspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/game_state" + servicespb "github.com/c4t-but-s4d/fastad/pkg/proto/data/services" + teamspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/teams" + receiverpb "github.com/c4t-but-s4d/fastad/pkg/proto/receiver" +) + +func Run(runCtx, shutdownCtx context.Context, cfg *receiver.Config) error { + db := cfg.Postgres.BunDB() + + producer, err := centutil.NewClientProducer( + cfg.CentrifugeClient.Address, + cfg.Channel, + cfg.Installation, + cfg.IntercomToken, + ) + if err != nil { + return fmt.Errorf("creating centrifuge producer: %w", err) + } + + dataServiceConn, err := grpcext.Dial( + cfg.DataService.Address, + cfg.Installation, + grpcext.AuthDialOptions(cfg.IntercomToken)..., + ) + if err != nil { + return fmt.Errorf("dialing data service: %w", err) + } + + teamsClient := teams.NewClient(teamspb.NewTeamsServiceClient(dataServiceConn), cfg.Installation) + servicesClient := services.NewClient(servicespb.NewServicesServiceClient(dataServiceConn), cfg.Installation) + gameStateClient := gamestate.NewClient(gspb.NewGameStateServiceClient(dataServiceConn), cfg.Installation) + + receiverMetrics := receiver.NewMetrics(cfg.Installation) + + receiverService := receiver.New( + db, + teamsClient, + servicesClient, + gameStateClient, + producer, + receiverMetrics, + ) + + if err := receiverService.RestoreState(runCtx); err != nil { + return fmt.Errorf("restoring state: %w", err) + } + + grpcServer := grpcext.NewServer( + grpcext.WithServerInstallation(cfg.Installation), + grpcext.WithServerTokenAuth(cfg.IntercomToken), + ) + receiverpb.RegisterReceiverServiceServer(grpcServer, receiverService) + + g, gctx := errgroup.WithContext(runCtx) + + if cfg.MetricsAddress != "" { + g.Go(func() error { + metrics.RunServer(runCtx, shutdownCtx, cfg.MetricsAddress) + return nil + }) + } + + g.Go(func() error { + if err := producer.Run(gctx); err != nil { + return fmt.Errorf("running client: %w", err) + } + return nil + }) + + g.Go(func() error { + if err := grpcext.RunServer(gctx, shutdownCtx, grpcServer, cfg.ListenAddress); err != nil { + return fmt.Errorf("running server: %w", err) + } + return nil + }) + + if err := g.Wait(); err != nil { + return fmt.Errorf("waiting: %w", err) + } + + return nil +} diff --git a/cmd/receiver/main.go b/cmd/receiver/main.go index 1fb5486..5c33e36 100644 --- a/cmd/receiver/main.go +++ b/cmd/receiver/main.go @@ -1,64 +1,24 @@ package main import ( - "context" - "errors" - "net/http" - "os" - "os/signal" - "syscall" - "time" + "go.uber.org/zap" - "github.com/c4t-but-s4d/fastad/internal/multiproto" - "github.com/c4t-but-s4d/fastad/internal/pinger" + "github.com/c4t-but-s4d/fastad/cmd/receiver/impl" "github.com/c4t-but-s4d/fastad/internal/receiver" - pingerpb "github.com/c4t-but-s4d/fastad/pkg/proto/pinger" - receiverpb "github.com/c4t-but-s4d/fastad/pkg/proto/receiver" - - "github.com/sirupsen/logrus" - "google.golang.org/grpc" - "google.golang.org/grpc/reflection" + "github.com/c4t-but-s4d/fastad/pkg/baseconfig" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/stop" ) func main() { - initLogger() - - logrus.Info("Starting flag receiver") - - grpcServer := grpc.NewServer() - receiverpb.RegisterReceiverServiceServer(grpcServer, receiver.New()) - pingerpb.RegisterPingerServiceServer(grpcServer, pinger.New()) - reflection.Register(grpcServer) - - httpServer := &http.Server{ - Addr: "0.0.0.0:8002", - Handler: multiproto.NewHandler(grpcServer), - } - - go func() { - logrus.Infof("Running http server on %s", httpServer.Addr) - if err := httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { - logrus.Fatalf("Error running http server: %v", err) - } - }() + defer logging.Init().Close() - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - <-c + cfg := baseconfig.MustSetupAll(&receiver.Config{}, baseconfig.WithEnvPrefix("FASTAD_RECEIVER")) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + runCtx, shutdownCtx, cancel := stop.SetupCtx() defer cancel() - if err := httpServer.Shutdown(ctx); err != nil { - logrus.Fatalf("Shutting down http server: %v", err) + if err := impl.Run(runCtx, shutdownCtx, cfg); err != nil { + zap.L().Fatal("receiver run failed", zap.Error(err)) } } - -func initLogger() { - mainFormatter := &logrus.TextFormatter{} - mainFormatter.FullTimestamp = true - mainFormatter.ForceColors = true - mainFormatter.PadLevelText = true - mainFormatter.TimestampFormat = "2006-01-02 15:04:05" - logrus.SetFormatter(mainFormatter) -} diff --git a/cmd/scheduler/impl/run.go b/cmd/scheduler/impl/run.go new file mode 100644 index 0000000..603f854 --- /dev/null +++ b/cmd/scheduler/impl/run.go @@ -0,0 +1,95 @@ +package impl + +import ( + "context" + "fmt" + + "go.temporal.io/sdk/client" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + "github.com/c4t-but-s4d/fastad/internal/scheduler" + "github.com/c4t-but-s4d/fastad/pkg/clients/gamestate" + "github.com/c4t-but-s4d/fastad/pkg/clients/services" + "github.com/c4t-but-s4d/fastad/pkg/clients/teams" + "github.com/c4t-but-s4d/fastad/pkg/grpcext" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/metrics" + gspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/game_state" + servicespb "github.com/c4t-but-s4d/fastad/pkg/proto/data/services" + teamspb "github.com/c4t-but-s4d/fastad/pkg/proto/data/teams" +) + +func Run(runCtx, shutdownCtx context.Context, cfg *scheduler.Config) error { + logger := zap.L().Named(cfg.Installation) + + temporalClient, err := client.Dial(client.Options{ + HostPort: cfg.Temporal.Address, + Logger: logging.NewTemporalAdapter( + logger.Named("temporal_client"), + ), + }) + if err != nil { + return fmt.Errorf("creating temporal client: %w", err) + } + defer temporalClient.Close() + + db := cfg.Postgres.BunDB() + + dataServiceConn, err := grpcext.Dial( + cfg.DataService.Address, + cfg.Installation, + grpcext.AuthDialOptions(cfg.IntercomToken)..., + ) + if err != nil { + return fmt.Errorf("dialing data service: %w", err) + } + + gameStateClient := gamestate.NewClient(gspb.NewGameStateServiceClient(dataServiceConn), cfg.Installation) + teamsClient := teams.NewClient(teamspb.NewTeamsServiceClient(dataServiceConn), cfg.Installation) + servicesClient := services.NewClient(servicespb.NewServicesServiceClient(dataServiceConn), cfg.Installation) + + t := scheduler.NewRoundScheduler( + temporalClient, + gameStateClient, + db, + logger, + ) + + cm := scheduler.NewCheckManager( + db, + gameStateClient, + teamsClient, + servicesClient, + temporalClient, + logger, + ) + + g, gctx := errgroup.WithContext(runCtx) + g.Go(func() error { + if err := t.Run(gctx); err != nil { + return fmt.Errorf("running round scheduler: %w", err) + } + return nil + }) + + g.Go(func() error { + if err := cm.Run(gctx); err != nil { + return fmt.Errorf("running check manager: %w", err) + } + return nil + }) + + if cfg.MetricsAddress != "" { + g.Go(func() error { + metrics.RunServer(gctx, shutdownCtx, cfg.MetricsAddress) + return nil + }) + } + + if err := g.Wait(); err != nil { + return fmt.Errorf("running errgroup: %w", err) + } + + return nil +} diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go new file mode 100644 index 0000000..b8ba74f --- /dev/null +++ b/cmd/scheduler/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/cmd/scheduler/impl" + "github.com/c4t-but-s4d/fastad/internal/scheduler" + "github.com/c4t-but-s4d/fastad/pkg/baseconfig" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/stop" +) + +func main() { + defer logging.Init().Close() + + cfg := baseconfig.MustSetupAll(&scheduler.Config{}, baseconfig.WithEnvPrefix("FASTAD_SCHEDULER")) + + runCtx, shutdownCtx, cancel := stop.SetupCtx() + defer cancel() + + if err := impl.Run(runCtx, shutdownCtx, cfg); err != nil { + zap.L().Fatal("scheduler worker run failed", zap.Error(err)) + } +} diff --git a/cmd/slac/impl/run.go b/cmd/slac/impl/run.go new file mode 100644 index 0000000..6b56f95 --- /dev/null +++ b/cmd/slac/impl/run.go @@ -0,0 +1,56 @@ +package impl + +import ( + "context" + "fmt" + + "golang.org/x/sync/errgroup" + + "github.com/c4t-but-s4d/fastad/internal/slac" + "github.com/c4t-but-s4d/fastad/pkg/grpcext" + "github.com/c4t-but-s4d/fastad/pkg/metrics" + slacpb "github.com/c4t-but-s4d/fastad/pkg/proto/slac" +) + +func Run(runCtx, shutdownCtx context.Context, cfg *slac.Config) error { + db := cfg.Postgres.BunDB() + + service := slac.NewService(db, cfg) + + if err := service.RestoreState(runCtx); err != nil { + return fmt.Errorf("restoring state: %w", err) + } + + g, gctx := errgroup.WithContext(runCtx) + + grpcServer := grpcext.NewServer( + grpcext.WithServerInstallation(cfg.Installation), + grpcext.WithServerTokenAuth(cfg.IntercomToken), + ) + slacpb.RegisterSlacServiceServer(grpcServer, service) + + if cfg.MetricsAddress != "" { + g.Go(func() error { + metrics.RunServer(gctx, shutdownCtx, cfg.MetricsAddress) + return nil + }) + } + + g.Go(func() error { + service.Run(gctx) + return nil + }) + + g.Go(func() error { + if err := grpcext.RunServer(gctx, shutdownCtx, grpcServer, cfg.ListenAddress); err != nil { + return fmt.Errorf("running server: %w", err) + } + return nil + }) + + if err := g.Wait(); err != nil { + return fmt.Errorf("waiting: %w", err) + } + + return nil +} diff --git a/cmd/slac/main.go b/cmd/slac/main.go new file mode 100644 index 0000000..bfe2508 --- /dev/null +++ b/cmd/slac/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "go.uber.org/zap" + + "github.com/c4t-but-s4d/fastad/cmd/slac/impl" + "github.com/c4t-but-s4d/fastad/internal/slac" + "github.com/c4t-but-s4d/fastad/pkg/baseconfig" + "github.com/c4t-but-s4d/fastad/pkg/logging" + "github.com/c4t-but-s4d/fastad/pkg/stop" +) + +func main() { + defer logging.Init().Close() + + cfg := baseconfig.MustSetupAll(&slac.Config{}, baseconfig.WithEnvPrefix("FASTAD_SLAC")) + + runCtx, shutdownCtx, cancel := stop.SetupCtx() + defer cancel() + + if err := impl.Run(runCtx, shutdownCtx, cfg); err != nil { + zap.L().Fatal("slac run failed", zap.Error(err)) + } +} diff --git a/config/game.yml b/config/game.yml new file mode 100644 index 0000000..8ac670b --- /dev/null +++ b/config/game.yml @@ -0,0 +1,32 @@ +game: + # start time in rfc3339 format + start_time: '2021-11-30T15:30:00+03:00' + end_time: '2022-11-30T15:30:00+03:00' + round_duration: '10s' + flag_lifetime_rounds: 10 + mode: 'classic' + hardness: 15 + +teams: + - name: 'team1' + address: 'addr3' + + - name: 'team2' + address: 'addr2' + +services: + - name: 'service1' + default_score: 3000 + checker: + default_timeout: 10s + path: 'service1/checker.py' + + - name: 'service2' + default_score: 2000 + checker: + default_timeout: 11s + path: 'service2/checker2.py' + actions: + get: + timeout: 12s + count: 2 diff --git a/config/tests/simple/game.yml b/config/tests/simple/game.yml new file mode 100644 index 0000000..497e661 --- /dev/null +++ b/config/tests/simple/game.yml @@ -0,0 +1,31 @@ +game: + start_time: '2025-01-12T20:50:00+03:00' + end_time: '2025-01-12T21:35:00+03:00' + round_duration: '10s' + flag_lifetime_rounds: 10 + hardness: 15 + checkers_base_path: '.' + +teams: + - name: 'team1' + address: '127.0.0.1' + + - name: 'team2' + address: '127.0.0.1' + +services: + - name: 'test service' + default_score: 3000 + checker: + default_timeout: 10s + path: 'tests/service/checker.py' + + - name: 'test service (no checker)' + default_score: 2000 + checker: + default_timeout: 11s + path: 'no-checker.py' + actions: + get: + timeout: 12s + count: 2 diff --git a/docker/presets/production/Caddyfile b/docker/presets/production/Caddyfile new file mode 100644 index 0000000..43f3080 --- /dev/null +++ b/docker/presets/production/Caddyfile @@ -0,0 +1,44 @@ +{ + servers :80 { + protocols h1 h2 h2c + } +} + +:80 { + log { + output stderr + level DEBUG + format console + } + + handle /temporal* { + reverse_proxy temporal-ui:8080 + } + + handle /api/* { + reverse_proxy api:8080 + } + + handle /centrifuge/* { + reverse_proxy api:8080 + } + + handle /healthcheck { + reverse_proxy api:8080 + } + + handle /flags { + reverse_proxy api:8080 + } + + handle /data.* { + reverse_proxy h2c://dataservice:8003 + } + + handle { + encode gzip + root * /front + try_files {path} /index.html + file_server + } +} diff --git a/docker/presets/production/checkers.Dockerfile b/docker/presets/production/checkers.Dockerfile new file mode 100644 index 0000000..f880082 --- /dev/null +++ b/docker/presets/production/checkers.Dockerfile @@ -0,0 +1,43 @@ +FROM golang:1.25-alpine AS builder + +ENV CGO_ENABLED=0 + +RUN apk add --no-cache upx + +WORKDIR /app + +COPY cmd cmd +COPY pkg pkg +COPY internal internal +COPY go.* ./ + +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + go build \ + -trimpath \ + -ldflags="-w -s" \ + -o "/checkers" \ + "./cmd/checkers/main.go" + +ARG COMPRESS_BINARIES="false" +ENV COMPRESS_BINARIES=$COMPRESS_BINARIES +RUN if [ "${COMPRESS_BINARIES}" = "true" ]; then upx --lzma -9 "/checkers"; fi + +FROM python:3.12-bookworm + +ENV PWNLIB_NOTERM=true +ENV UV_BREAK_SYSTEM_PACKAGES=true +ENV PIP_BREAK_SYSTEM_PACKAGES=true + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/ + +ARG CHECKERS_DIR +COPY ${CHECKERS_DIR} /checkers-scripts +RUN --mount=type=cache,target=/root/.cache/uv \ + find /checkers-scripts -name "requirements.txt" | while read -r file; do \ + echo "Installing requirements from ${file}"; uv pip install --system -r "$file"; \ + done + +COPY --from=builder /checkers /checkers + +ENTRYPOINT ["/checkers"] diff --git a/docker/presets/production/compose.yml b/docker/presets/production/compose.yml new file mode 100644 index 0000000..8a5cc49 --- /dev/null +++ b/docker/presets/production/compose.yml @@ -0,0 +1,312 @@ +services: + migrator: + build: + context: . + dockerfile: ./docker/presets/production/service.Dockerfile + args: + SERVICE_NAME: migrator + restart: on-failure:5 + environment: + FASTAD_MIGRATOR_POSTGRES_DSN: ${FASTAD_POSTGRES_DSN} + depends_on: + postgres: + condition: service_healthy + command: ["sh", "-c", "/service init && /service migrate"] + + dataservice: + build: + context: . + dockerfile: ./docker/presets/production/service.Dockerfile + args: + SERVICE_NAME: dataservice + restart: unless-stopped + environment: + FASTAD_DATA_SERVICE_POSTGRES_DSN: ${FASTAD_POSTGRES_DSN} + FASTAD_DATA_SERVICE_INTERCOM_TOKEN: ${FASTAD_INTERCOM_TOKEN} + FASTAD_DATA_SERVICE_LISTEN_ADDRESS: ":8003" + FASTAD_DATA_SERVICE_METRICS_ADDRESS: ":3004" + FASTAD_DATA_SERVICE_INSTALLATION: "fastad-prod" + depends_on: + migrator: + condition: service_completed_successfully + healthcheck: + test: ['CMD-SHELL', 'nc -z localhost 8003 || exit 1'] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + + api: + build: + context: . + dockerfile: ./docker/presets/production/service.Dockerfile + args: + SERVICE_NAME: api + restart: unless-stopped + environment: + FASTAD_API_POSTGRES_DSN: ${FASTAD_POSTGRES_DSN} + FASTAD_API_INTERCOM_TOKEN: ${FASTAD_INTERCOM_TOKEN} + FASTAD_API_LISTEN_ADDRESS: ":8080" + FASTAD_API_METRICS_ADDRESS: ":3001" + FASTAD_API_DATA_SERVICE_ADDRESS: "dataservice:8003" + FASTAD_API_RECEIVER_ADDRESS: "receiver:8001" + FASTAD_API_SLAC_ADDRESS: "slac:8002" + FASTAD_API_INSTALLATION: "fastad-prod" + FASTAD_API_SCOREBOARD_CHANNEL: "scoreboard" + depends_on: + dataservice: + condition: service_healthy + slac: + condition: service_healthy + healthcheck: + test: ['CMD-SHELL', 'wget -qO- http://localhost:8080/healthcheck || exit 1'] + interval: 5s + timeout: 5s + retries: 10 + start_period: 30s + + receiver: + build: + context: . + dockerfile: ./docker/presets/production/service.Dockerfile + args: + SERVICE_NAME: receiver + restart: unless-stopped + environment: + FASTAD_RECEIVER_POSTGRES_DSN: ${FASTAD_POSTGRES_DSN} + FASTAD_RECEIVER_INTERCOM_TOKEN: ${FASTAD_INTERCOM_TOKEN} + FASTAD_RECEIVER_LISTEN_ADDRESS: ":8001" + FASTAD_RECEIVER_METRICS_ADDRESS: ":3002" + FASTAD_RECEIVER_DATA_SERVICE_ADDRESS: "dataservice:8003" + FASTAD_RECEIVER_CENTRIFUGE_CLIENT_ADDRESS: "ws://api:8080/centrifuge/websocket" + FASTAD_RECEIVER_INSTALLATION: "fastad-prod" + depends_on: + dataservice: + condition: service_healthy + api: + condition: service_healthy + healthcheck: + test: ['CMD-SHELL', 'nc -z localhost 8001 || exit 1'] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + + slac: + build: + context: . + dockerfile: ./docker/presets/production/service.Dockerfile + args: + SERVICE_NAME: slac + restart: unless-stopped + environment: + FASTAD_SLAC_POSTGRES_DSN: ${FASTAD_POSTGRES_DSN} + FASTAD_SLAC_INTERCOM_TOKEN: ${FASTAD_INTERCOM_TOKEN} + FASTAD_SLAC_LISTEN_ADDRESS: ":8002" + FASTAD_SLAC_METRICS_ADDRESS: ":3003" + FASTAD_SLAC_DATA_SERVICE_ADDRESS: "dataservice:8003" + FASTAD_SLAC_INSTALLATION: "fastad-prod" + depends_on: + dataservice: + condition: service_healthy + healthcheck: + test: ['CMD-SHELL', 'nc -z localhost 8002 || exit 1'] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + + scheduler: + build: + context: . + dockerfile: ./docker/presets/production/service.Dockerfile + args: + SERVICE_NAME: scheduler + restart: unless-stopped + environment: + FASTAD_SCHEDULER_POSTGRES_DSN: ${FASTAD_POSTGRES_DSN} + FASTAD_SCHEDULER_INTERCOM_TOKEN: ${FASTAD_INTERCOM_TOKEN} + FASTAD_SCHEDULER_METRICS_ADDRESS: ":3005" + FASTAD_SCHEDULER_DATA_SERVICE_ADDRESS: "dataservice:8003" + FASTAD_SCHEDULER_TEMPORAL_ADDRESS: "temporal:7233" + FASTAD_SCHEDULER_INSTALLATION: "fastad-prod" + depends_on: + temporal-create-namespace: + condition: service_completed_successfully + dataservice: + condition: service_healthy + + checkers: + build: + context: . + dockerfile: ./docker/presets/production/checkers.Dockerfile + args: + CHECKERS_DIR: ${CHECKERS_DIR:-checkers} + restart: unless-stopped + environment: + FASTAD_CHECKERS_POSTGRES_DSN: ${FASTAD_POSTGRES_DSN} + FASTAD_CHECKERS_INTERCOM_TOKEN: ${FASTAD_INTERCOM_TOKEN} + FASTAD_CHECKERS_METRICS_ADDRESS: ":3006" + FASTAD_CHECKERS_TEMPORAL_ADDRESS: "temporal:7233" + FASTAD_CHECKERS_DATA_SERVICE_ADDRESS: "dataservice:8003" + FASTAD_CHECKERS_INSTALLATION: "fastad-prod" + depends_on: + temporal-create-namespace: + condition: service_completed_successfully + dataservice: + condition: service_healthy + + caddy: + build: + context: . + dockerfile: ./docker/presets/simple/caddy.Dockerfile + restart: unless-stopped + ports: + - "${CADDY_PORT}:80" + volumes: + - ./docker/presets/production/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + depends_on: + api: + condition: service_healthy + + temporal-admin-tools: + image: temporalio/admin-tools:1.26.2 + restart: on-failure:6 + depends_on: + temporal-postgres: + condition: service_healthy + environment: + DB: postgres12 + DB_PORT: ${TEMPORAL_POSTGRES_PORT} + POSTGRES_USER: ${TEMPORAL_POSTGRES_USER} + POSTGRES_PWD: ${TEMPORAL_POSTGRES_PASSWORD} + POSTGRES_SEEDS: ${TEMPORAL_POSTGRES_HOST} + SQL_PASSWORD: ${TEMPORAL_POSTGRES_PASSWORD} + volumes: + - ./docker/presets/simple/temporal-scripts:/scripts:ro + entrypoint: ["/bin/sh"] + command: /scripts/setup-postgres.sh + + temporal: + image: temporalio/server:1.26.2 + restart: unless-stopped + environment: + DB: postgres12 + DB_PORT: ${TEMPORAL_POSTGRES_PORT} + POSTGRES_USER: ${TEMPORAL_POSTGRES_USER} + POSTGRES_PWD: ${TEMPORAL_POSTGRES_PASSWORD} + POSTGRES_SEEDS: ${TEMPORAL_POSTGRES_HOST} + BIND_ON_IP: "0.0.0.0" + DYNAMIC_CONFIG_FILE_PATH: config/dynamicconfig/development-sql.yaml + depends_on: + temporal-admin-tools: + condition: service_completed_successfully + volumes: + - ./docker/presets/simple/temporal-dynamicconfig:/etc/temporal/config/dynamicconfig:ro + healthcheck: + test: ['CMD', 'nc', '-z', 'localhost', '7233'] + interval: 5s + timeout: 3s + retries: 60 + start_period: 30s + + temporal-create-namespace: + image: temporalio/admin-tools:1.26.2 + restart: on-failure:5 + depends_on: + temporal: + condition: service_healthy + environment: + TEMPORAL_ADDRESS: temporal:7233 + DEFAULT_NAMESPACE: default + volumes: + - ./docker/presets/simple/temporal-scripts:/scripts:ro + entrypoint: ["/bin/sh"] + command: /scripts/create-namespace.sh + + temporal-ui: + image: temporalio/ui:2.33.0 + restart: unless-stopped + environment: + - TEMPORAL_ADDRESS=temporal:7233 + - TEMPORAL_UI_PORT=8080 + - TEMPORAL_UI_PUBLIC_PATH=/temporal + depends_on: + temporal: + condition: service_healthy + + postgres: + image: postgres:17.2-alpine + restart: unless-stopped + ports: + - "127.0.0.1:${FASTAD_POSTGRES_EXTERNAL_PORT:-5433}:5432" + environment: + POSTGRES_USER: fastad + POSTGRES_PASSWORD: fastad + POSTGRES_DB: fastad + volumes: + - ${FASTAD_DATA_DIR:-./data}/fastad-db:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U fastad -d fastad + interval: 5s + timeout: 5s + retries: 5 + start_period: 30s + + temporal-postgres: + image: postgres:17.2-alpine + restart: unless-stopped + environment: + POSTGRES_USER: ${TEMPORAL_POSTGRES_USER} + POSTGRES_PASSWORD: ${TEMPORAL_POSTGRES_PASSWORD} + POSTGRES_DB: ${TEMPORAL_POSTGRES_DATABASE} + volumes: + - ${FASTAD_DATA_DIR:-./data}/temporal-db:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U temporal -d temporal + interval: 5s + timeout: 5s + retries: 5 + start_period: 30s + + victoriametrics: + image: victoriametrics/victoria-metrics:v1.108.1 + restart: unless-stopped + command: + - "-storageDataPath=/storage" + - "-retentionPeriod=30d" + - "-promscrape.config=/etc/victoriametrics/scrape.yml" + volumes: + - ${FASTAD_DATA_DIR:-./data}/victoriametrics:/storage + - ./docker/presets/production/victoriametrics:/etc/victoriametrics:ro + healthcheck: + test: ['CMD', 'wget', '-qO-', 'http://127.0.0.1:8428/-/healthy'] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + + grafana: + image: grafana/grafana:11.4.0 + restart: unless-stopped + ports: + - "${GRAFANA_PORT:-3030}:3000" + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-fastad} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + volumes: + - grafana-data:/var/lib/grafana + - ./docker/presets/production/grafana/provisioning:/etc/grafana/provisioning:ro + - ./docker/presets/production/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + victoriametrics: + condition: service_healthy + +volumes: + caddy-data: + caddy-config: + grafana-data: diff --git a/docker/presets/production/grafana/dashboards/checkers.json b/docker/presets/production/grafana/dashboards/checkers.json new file mode 100644 index 0000000..797a974 --- /dev/null +++ b/docker/presets/production/grafana/dashboards/checkers.json @@ -0,0 +1,438 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*STATUS_UP.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*STATUS_DOWN.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*STATUS_MUMBLE.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*STATUS_CORRUPT.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(fastad_checker_execution_status_total[1m])) by (status)", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Checker Execution Status (rate/min)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(fastad_checker_executions_total[1m])) by (action)", + "legendFormat": "{{action}}", + "refId": "A" + } + ], + "title": "Checker Executions by Action (rate/min)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(fastad_checker_executions_total[1m])) by (service_id)", + "legendFormat": "service {{service_id}}", + "refId": "A" + } + ], + "title": "Checker Executions by Service (rate/min)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.8 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(fastad_checker_execution_status_total{status=\"STATUS_UP\"}[5m])) by (service_id) / sum(rate(fastad_checker_execution_status_total[5m])) by (service_id)", + "legendFormat": "service {{service_id}}", + "refId": "A" + } + ], + "title": "Service Availability (UP ratio)", + "type": "gauge" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": ["fastad", "checkers"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "FastAD Checkers", + "uid": "fastad-checkers", + "version": 1 +} diff --git a/docker/presets/production/grafana/dashboards/flags.json b/docker/presets/production/grafana/dashboards/flags.json new file mode 100644 index 0000000..173efbe --- /dev/null +++ b/docker/presets/production/grafana/dashboards/flags.json @@ -0,0 +1,932 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(fastad_flags_processed_total[1m])) by (verdict)", + "legendFormat": "{{verdict}}", + "refId": "A" + } + ], + "title": "Flags Processed by Verdict (rate/min)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(fastad_attacks_by_attacker_total[1m])) by (service_name)", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "title": "Successful Attacks by Service (rate/min)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(fastad_flags_submitted_total[1m])) by (team_name)", + "legendFormat": "{{team_name}}", + "refId": "A" + } + ], + "title": "Flag Submissions by Team (rate/min)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum(rate(fastad_flag_processing_seconds_bucket[5m])) by (le, verdict_summary))", + "legendFormat": "p50 {{verdict_summary}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(fastad_flag_processing_seconds_bucket[5m])) by (le, verdict_summary))", + "legendFormat": "p95 {{verdict_summary}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum(rate(fastad_flag_processing_seconds_bucket[5m])) by (le, verdict_summary))", + "legendFormat": "p99 {{verdict_summary}}", + "refId": "C" + } + ], + "title": "Flag Processing Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(fastad_points_transferred_total[5m])) by (service_name)", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "title": "Points Transferred by Service (5m window)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 10, + "panels": [], + "title": "Detailed Attack Analysis", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 6, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(10, sum by (attacker_name) (increase(fastad_flag_verdicts_detailed_total{verdict=\"VERDICT_ACCEPTED\", attacker_name=~\"$attacker\", victim_name=~\"$victim\", service_name=~\"$service\"}[$__range])))", + "format": "time_series", + "instant": true, + "legendFormat": "{{attacker_name}}", + "refId": "A" + } + ], + "title": "Top 10 Attackers (Successful Attacks)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 7, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(10, sum by (victim_name) (increase(fastad_flag_verdicts_detailed_total{verdict=\"VERDICT_ACCEPTED\", attacker_name=~\"$attacker\", victim_name=~\"$victim\", service_name=~\"$service\"}[$__range])))", + "format": "time_series", + "instant": true, + "legendFormat": "{{victim_name}}", + "refId": "A" + } + ], + "title": "Top 10 Victims (Attacks Received)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "color-background" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "green", + "value": 1 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "orange", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "attacker_name" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "victim_name" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "service_name" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + }, + { + "id": "custom.width", + "value": 120 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (attacker_name, victim_name, service_name) (increase(fastad_flag_verdicts_detailed_total{verdict=\"VERDICT_ACCEPTED\", attacker_name=~\"$attacker\", victim_name=~\"$victim\", service_name=~\"$service\"}[$__range]))", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Attack Matrix (Attacker → Victim by Service)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "Time": 0, + "Value": 4, + "attacker_name": 1, + "service_name": 3, + "victim_name": 2 + }, + "renameByName": { + "Value": "Attacks", + "attacker_name": "Attacker", + "service_name": "Service", + "victim_name": "Victim" + } + } + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "greater", + "options": { + "value": 0 + } + }, + "fieldName": "Attacks" + } + ], + "match": "any", + "type": "include" + } + } + ], + "type": "table" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": ["fastad", "flags"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(fastad_flag_verdicts_detailed_total, attacker_name)", + "hide": 0, + "includeAll": true, + "label": "Attacker", + "multi": true, + "name": "attacker", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fastad_flag_verdicts_detailed_total, attacker_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(fastad_flag_verdicts_detailed_total, victim_name)", + "hide": 0, + "includeAll": true, + "label": "Victim", + "multi": true, + "name": "victim", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fastad_flag_verdicts_detailed_total, victim_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(fastad_flag_verdicts_detailed_total, service_name)", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fastad_flag_verdicts_detailed_total, service_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "FastAD Flag Submissions", + "uid": "fastad-flags", + "version": 1 +} diff --git a/docker/presets/production/grafana/dashboards/overview.json b/docker/presets/production/grafana/dashboards/overview.json new file mode 100644 index 0000000..7f3f322 --- /dev/null +++ b/docker/presets/production/grafana/dashboards/overview.json @@ -0,0 +1,460 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(http_requests_total[1m])) by (url)", + "legendFormat": "{{url}}", + "refId": "A" + } + ], + "title": "HTTP Request Rate by URL", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, url))", + "legendFormat": "p95 {{url}}", + "refId": "A" + } + ], + "title": "HTTP Latency p95 by URL", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(grpc_server_handled_total[1m])) by (grpc_service, grpc_method)", + "legendFormat": "{{grpc_service}}/{{grpc_method}}", + "refId": "A" + } + ], + "title": "gRPC Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket[5m])) by (le, grpc_service, grpc_method))", + "legendFormat": "p95 {{grpc_service}}/{{grpc_method}}", + "refId": "A" + } + ], + "title": "gRPC Latency p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(grpc_server_panics_recovered_total)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "gRPC Panics Total", + "type": "stat" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": ["fastad"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "FastAD Overview", + "uid": "fastad-overview", + "version": 1 +} diff --git a/docker/presets/production/grafana/provisioning/dashboards/default.yml b/docker/presets/production/grafana/provisioning/dashboards/default.yml new file mode 100644 index 0000000..96a4ca6 --- /dev/null +++ b/docker/presets/production/grafana/provisioning/dashboards/default.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: 'FastAD Dashboards' + orgId: 1 + folder: 'FastAD' + folderUid: 'fastad' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/docker/presets/production/grafana/provisioning/datasources/victoriametrics.yml b/docker/presets/production/grafana/provisioning/datasources/victoriametrics.yml new file mode 100644 index 0000000..9c578b5 --- /dev/null +++ b/docker/presets/production/grafana/provisioning/datasources/victoriametrics.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: VictoriaMetrics + type: prometheus + access: proxy + url: http://victoriametrics:8428 + isDefault: true + editable: false diff --git a/docker/presets/production/service.Dockerfile b/docker/presets/production/service.Dockerfile new file mode 100644 index 0000000..7b20e76 --- /dev/null +++ b/docker/presets/production/service.Dockerfile @@ -0,0 +1,31 @@ +FROM golang:1.25-alpine AS builder + +ARG SERVICE_NAME +ENV CGO_ENABLED=0 + +RUN apk add --no-cache upx + +WORKDIR /app + +COPY cmd cmd +COPY pkg pkg +COPY internal internal +COPY go.* ./ + +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + go build \ + -trimpath \ + -ldflags="-w -s" \ + -o "/service" \ + "./cmd/${SERVICE_NAME}/main.go" + +ARG COMPRESS_BINARIES="false" +ENV COMPRESS_BINARIES=$COMPRESS_BINARIES +RUN if [ "${COMPRESS_BINARIES}" = "true" ]; then upx --lzma -9 "/service"; fi + +FROM alpine:3.21 + +COPY --from=builder /service /service + +CMD ["/service"] diff --git a/docker/presets/production/victoriametrics/scrape.yml b/docker/presets/production/victoriametrics/scrape.yml new file mode 100644 index 0000000..00c244c --- /dev/null +++ b/docker/presets/production/victoriametrics/scrape.yml @@ -0,0 +1,28 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + +scrape_configs: + - job_name: 'api' + static_configs: + - targets: ['api:3001'] + + - job_name: 'receiver' + static_configs: + - targets: ['receiver:3002'] + + - job_name: 'slac' + static_configs: + - targets: ['slac:3003'] + + - job_name: 'dataservice' + static_configs: + - targets: ['dataservice:3004'] + + - job_name: 'scheduler' + static_configs: + - targets: ['scheduler:3005'] + + - job_name: 'checkers' + static_configs: + - targets: ['checkers:3006'] diff --git a/docker/presets/simple/Caddyfile b/docker/presets/simple/Caddyfile new file mode 100644 index 0000000..95d3749 --- /dev/null +++ b/docker/presets/simple/Caddyfile @@ -0,0 +1,44 @@ +{ + servers :80 { + protocols h1 h2 h2c + } +} + +:80 { + log { + output stderr + level DEBUG + format console + } + + handle /temporal* { + reverse_proxy temporal-ui:8080 + } + + handle /api/* { + reverse_proxy app:8080 + } + + handle /centrifuge/* { + reverse_proxy app:8080 + } + + handle /healthcheck { + reverse_proxy app:8080 + } + + handle /flags { + reverse_proxy app:8080 + } + + handle /data.* { + reverse_proxy h2c://app:8004 + } + + handle { + encode gzip + root * /front + try_files {path} /index.html + file_server + } +} diff --git a/docker/presets/simple/app.Dockerfile b/docker/presets/simple/app.Dockerfile new file mode 100644 index 0000000..b4b6b97 --- /dev/null +++ b/docker/presets/simple/app.Dockerfile @@ -0,0 +1,52 @@ +FROM golang:1.25-alpine AS builder + +ENV CGO_ENABLED=0 + +RUN apk add --no-cache upx + +WORKDIR /app + +COPY cmd cmd +COPY pkg pkg +COPY internal internal +COPY go.* ./ + +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + go build \ + -trimpath \ + -ldflags="-w -s" \ + -o "/allinone" \ + "./cmd/allinone/main.go" + +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + go build \ + -trimpath \ + -ldflags="-w -s" \ + -o "/migrator" \ + "./cmd/migrator/main.go" + +ARG COMPRESS_BINARIES="false" +ENV COMPRESS_BINARIES=$COMPRESS_BINARIES +RUN if [ "${COMPRESS_BINARIES}" = "true" ]; then upx --lzma -9 "/allinone" && upx --lzma -9 "/migrator"; fi + +FROM python:3.12-bookworm + +ENV PWNLIB_NOTERM=true +ENV UV_BREAK_SYSTEM_PACKAGES=true +ENV PIP_BREAK_SYSTEM_PACKAGES=true + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/ + +ARG CHECKERS_DIR +COPY ${CHECKERS_DIR} /checkers +RUN --mount=type=cache,target=/root/.cache/uv \ + find /checkers -name "requirements.txt" | while read -r file; do \ + echo "Installing requirements from ${file}"; uv pip install --system -r "$file"; \ + done + +COPY --from=builder /allinone /allinone +COPY --from=builder /migrator /migrator + +CMD ["/bin/sh", "-c", "/migrator init && /migrator migrate && /allinone"] diff --git a/docker/presets/simple/caddy.Dockerfile b/docker/presets/simple/caddy.Dockerfile new file mode 100644 index 0000000..8991d24 --- /dev/null +++ b/docker/presets/simple/caddy.Dockerfile @@ -0,0 +1,16 @@ +FROM node:22.14.0-alpine AS front-base + +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" + +COPY front /app +WORKDIR /app +RUN corepack enable + +FROM front-base AS front-build +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +RUN pnpm run build + +FROM caddy:2.9.1-alpine + +COPY --from=front-build /app/dist /front diff --git a/docker/presets/simple/compose.yml b/docker/presets/simple/compose.yml new file mode 100644 index 0000000..9c4ad72 --- /dev/null +++ b/docker/presets/simple/compose.yml @@ -0,0 +1,143 @@ +services: + app: + build: + context: . + dockerfile: ./docker/presets/simple/app.Dockerfile + restart: unless-stopped + environment: + FASTAD_ALLINONE_POSTGRES_DSN: ${FASTAD_POSTGRES_DSN} + FASTAD_MIGRATOR_POSTGRES_DSN: ${FASTAD_POSTGRES_DSN} + FASTAD_ALLINONE_INTERCOM_TOKEN: ${FASTAD_INTERCOM_TOKEN} + FASTAD_ALLINONE_TEMPORAL_ADDRESS: "temporal:7233" + FASTAD_ALLINONE_API_LISTEN_ADDRESS: ":8080" + FASTAD_LOG_LEVEL: ${FASTAD_LOG_LEVEL} + depends_on: + temporal-create-namespace: + condition: service_completed_successfully + postgres: + condition: service_healthy + healthcheck: + test: ['CMD-SHELL', 'wget -qO- http://localhost:8080/healthcheck || exit 1'] + interval: 5s + timeout: 5s + retries: 10 + start_period: 30s + + caddy: + build: + context: . + dockerfile: ./docker/presets/simple/caddy.Dockerfile + restart: unless-stopped + ports: + - "${CADDY_PORT}:80" + volumes: + - ./docker/presets/simple/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + depends_on: + app: + condition: service_healthy + + temporal-admin-tools: + image: temporalio/admin-tools:1.26.2 + restart: on-failure:6 + depends_on: + temporal-postgres: + condition: service_healthy + environment: + DB: postgres12 + DB_PORT: ${TEMPORAL_POSTGRES_PORT} + POSTGRES_USER: ${TEMPORAL_POSTGRES_USER} + POSTGRES_PWD: ${TEMPORAL_POSTGRES_PASSWORD} + POSTGRES_SEEDS: ${TEMPORAL_POSTGRES_HOST} + SQL_PASSWORD: ${TEMPORAL_POSTGRES_PASSWORD} + volumes: + - ./docker/presets/simple/temporal-scripts:/scripts:ro + entrypoint: ["/bin/sh"] + command: /scripts/setup-postgres.sh + + temporal: + image: temporalio/server:1.26.2 + restart: unless-stopped + environment: + DB: postgres12 + DB_PORT: ${TEMPORAL_POSTGRES_PORT} + POSTGRES_USER: ${TEMPORAL_POSTGRES_USER} + POSTGRES_PWD: ${TEMPORAL_POSTGRES_PASSWORD} + POSTGRES_SEEDS: ${TEMPORAL_POSTGRES_HOST} + BIND_ON_IP: "0.0.0.0" + DYNAMIC_CONFIG_FILE_PATH: config/dynamicconfig/development-sql.yaml + depends_on: + temporal-admin-tools: + condition: service_completed_successfully + volumes: + - ./docker/presets/simple/temporal-dynamicconfig:/etc/temporal/config/dynamicconfig:ro + healthcheck: + test: ['CMD', 'nc', '-z', 'localhost', '7233'] + interval: 5s + timeout: 3s + retries: 60 + start_period: 30s + + temporal-create-namespace: + image: temporalio/admin-tools:1.26.2 + restart: on-failure:5 + depends_on: + temporal: + condition: service_healthy + environment: + TEMPORAL_ADDRESS: temporal:7233 + DEFAULT_NAMESPACE: default + volumes: + - ./docker/presets/simple/temporal-scripts:/scripts:ro + entrypoint: ["/bin/sh"] + command: /scripts/create-namespace.sh + + temporal-ui: + image: temporalio/ui:2.33.0 + restart: unless-stopped + environment: + - TEMPORAL_ADDRESS=temporal:7233 + - TEMPORAL_UI_PORT=8080 + - TEMPORAL_UI_PUBLIC_PATH=/temporal + depends_on: + temporal: + condition: service_healthy + + postgres: + image: postgres:17.2-alpine + restart: unless-stopped + ports: + - "127.0.0.1:${FASTAD_POSTGRES_EXTERNAL_PORT:-5433}:5432" + environment: + POSTGRES_USER: fastad + POSTGRES_PASSWORD: fastad + POSTGRES_DB: fastad + volumes: + - ${FASTAD_DATA_DIR:-./data}/fastad-db:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U fastad -d fastad + interval: 5s + timeout: 5s + retries: 5 + start_period: 30s + + temporal-postgres: + image: postgres:17.2-alpine + restart: unless-stopped + environment: + POSTGRES_USER: ${TEMPORAL_POSTGRES_USER} + POSTGRES_PASSWORD: ${TEMPORAL_POSTGRES_PASSWORD} + POSTGRES_DB: ${TEMPORAL_POSTGRES_DATABASE} + volumes: + - ${FASTAD_DATA_DIR:-./data}/temporal-db:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U temporal -d temporal + interval: 5s + timeout: 5s + retries: 5 + start_period: 30s + +volumes: + caddy-data: + caddy-config: diff --git a/docker/presets/simple/temporal-dynamicconfig/development-sql.yaml b/docker/presets/simple/temporal-dynamicconfig/development-sql.yaml new file mode 100644 index 0000000..8862dfa --- /dev/null +++ b/docker/presets/simple/temporal-dynamicconfig/development-sql.yaml @@ -0,0 +1,6 @@ +limit.maxIDLength: + - value: 255 + constraints: {} +system.forceSearchAttributesCacheRefreshOnRead: + - value: true # Dev setup only. Please don't turn this on in production. + constraints: {} diff --git a/docker/presets/simple/temporal-scripts/create-namespace.sh b/docker/presets/simple/temporal-scripts/create-namespace.sh new file mode 100755 index 0000000..45334f0 --- /dev/null +++ b/docker/presets/simple/temporal-scripts/create-namespace.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +NAMESPACE=${DEFAULT_NAMESPACE:-default} +TEMPORAL_ADDRESS=${TEMPORAL_ADDRESS:-temporal:7233} + +echo "Waiting for Temporal server port to be available..." +nc -z -w 10 $(echo $TEMPORAL_ADDRESS | cut -d: -f1) $(echo $TEMPORAL_ADDRESS | cut -d: -f2) +echo 'Temporal server port is available' + +echo 'Waiting for Temporal server to be healthy...' +max_attempts=3 +attempt=0 + +until temporal operator cluster health --address $TEMPORAL_ADDRESS; do + attempt=$((attempt + 1)) + if [ $attempt -ge $max_attempts ]; then + echo "Server did not become healthy after $max_attempts attempts" + exit 1 + fi + echo "Server not ready yet, waiting... (attempt $attempt/$max_attempts)" + sleep 5 +done + +echo "Server is healthy, creating namespace '$NAMESPACE'..." +temporal operator namespace describe -n $NAMESPACE --address $TEMPORAL_ADDRESS || temporal operator namespace create -n $NAMESPACE --address $TEMPORAL_ADDRESS +echo "Namespace '$NAMESPACE' created" diff --git a/docker/presets/simple/temporal-scripts/setup-postgres.sh b/docker/presets/simple/temporal-scripts/setup-postgres.sh new file mode 100755 index 0000000..4c6f8ca --- /dev/null +++ b/docker/presets/simple/temporal-scripts/setup-postgres.sh @@ -0,0 +1,33 @@ +#!/bin/sh +set -eu + +echo 'Starting PostgreSQL schema setup for Temporal...' +echo 'Waiting for PostgreSQL port to be available...' + +# Wait for postgres to be reachable +max_attempts=30 +attempt=0 +until nc -z -w 5 ${POSTGRES_SEEDS} ${DB_PORT}; do + attempt=$((attempt + 1)) + if [ $attempt -ge $max_attempts ]; then + echo "PostgreSQL did not become available after $max_attempts attempts" + exit 1 + fi + echo "Waiting for PostgreSQL... (attempt $attempt/$max_attempts)" + sleep 2 +done +echo 'PostgreSQL port is available' + +# Create and setup temporal database +echo 'Creating temporal database...' +temporal-sql-tool --plugin postgres12 --ep ${POSTGRES_SEEDS} -u ${POSTGRES_USER} -p ${DB_PORT} --db temporal create || echo 'Database temporal may already exist' +temporal-sql-tool --plugin postgres12 --ep ${POSTGRES_SEEDS} -u ${POSTGRES_USER} -p ${DB_PORT} --db temporal setup-schema -v 0.0 +temporal-sql-tool --plugin postgres12 --ep ${POSTGRES_SEEDS} -u ${POSTGRES_USER} -p ${DB_PORT} --db temporal update-schema -d /etc/temporal/schema/postgresql/v12/temporal/versioned + +# Create and setup visibility database +echo 'Creating temporal_visibility database...' +temporal-sql-tool --plugin postgres12 --ep ${POSTGRES_SEEDS} -u ${POSTGRES_USER} -p ${DB_PORT} --db temporal_visibility create || echo 'Database temporal_visibility may already exist' +temporal-sql-tool --plugin postgres12 --ep ${POSTGRES_SEEDS} -u ${POSTGRES_USER} -p ${DB_PORT} --db temporal_visibility setup-schema -v 0.0 +temporal-sql-tool --plugin postgres12 --ep ${POSTGRES_SEEDS} -u ${POSTGRES_USER} -p ${DB_PORT} --db temporal_visibility update-schema -d /etc/temporal/schema/postgresql/v12/visibility/versioned + +echo 'PostgreSQL schema setup complete' diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..a942ea8 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,699 @@ + + +
+ + +Interactive heatmap showing rating delta based on attacker and victim scores
+// FastAD rating delta calculation +const scale = 50 * Math.sqrt(hardness); +const norm = Math.log(Math.log(hardness)) / 12; +const ratingDelta = Math.sqrt(attackerScore) - Math.sqrt(victimScore); +const ratingDeltaNorm = ratingDelta * norm; +const attackerDelta = scale / (1 + Math.exp(ratingDeltaNorm));+