feat: Introduce Typesense-Powered Search, RAG Chat, and Platform Upgrades

Dockerfile

@@ -22,7 +22,7 @@ RUN --mount=type=cache,target=/tmp/lock,sharing=locked \
 #########################################################################################################
 # Backend Build
 #########################################################################################################
-FROM golang:1.21 as backend-build
+FROM golang:1.22 as backend-build
 
 WORKDIR /go/src/github.com/fastenhealth/fasten-onprem
 COPY . .

backend/cmd/fasten/fasten.go

@@ -3,20 +3,22 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
+	"log"
+	"os"
+	"time"
+
 	"github.com/analogj/go-util/utils"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/config"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/database"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/errors"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/event_bus"
+	"github.com/fastenhealth/fasten-onprem/backend/pkg/search"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/version"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/web"
 	"github.com/fastenhealth/fasten-onprem/backend/resources"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
-	"io"
-	"log"
-	"os"
-	"time"
 )
 
 var goos string
@@ -111,6 +113,18 @@ func main() {
 						}
 					}()
 
+					// Check if Typesense (search) is enabled and initialize it
+					if appconfig.IsSet("search") && appconfig.GetString("search.uri") != "" {
+						err = search.Init(appconfig, appLogger)
+						if err != nil {
+							appLogger.Error("failed to initialize Typesense: %w", err)
+						}
+
+						appLogger.Info("Typesense initialized successfully")
+					} else {
+						appLogger.Info("Search is disabled, skipping Typesense initialization.")
+					}
+
 					settingsData, err := json.Marshal(appconfig.AllSettings())
 					appLogger.Debug(string(settingsData), err)
 

backend/pkg/config/config.go

@@ -94,5 +94,46 @@ func (c *configuration) ValidateConfig() error {
 			return errors.ConfigValidationError("database.encryption.key must be at least 10 characters")
 		}
 	}
+
+	if c.IsSet("search") {
+		key := c.GetString("search.uri")
+		if key == "" {
+			return errors.ConfigValidationError("search.uri cannot be empty")
+		}
+		key = c.GetString("search.api_key")
+		if key == "" {
+			return errors.ConfigValidationError("search.api_key cannot be empty")
+		}
+		key = c.GetString("search.collection_name")
+		if key == "" {
+			return errors.ConfigValidationError("search.collection_name cannot be empty")
+		}
+	}
+
+	if c.IsSet("search.chat") {
+		key := c.GetString("search.chat.conversation_collection_name")
+		if key == "" {
+			return errors.ConfigValidationError("search.chat.conversation_collection_name cannot be empty")
+		}
+
+		key = c.GetString("search.chat.model.id")
+		if key == "" {
+			return errors.ConfigValidationError("search.chat.model.id cannot be empty")
+		}
+		key = c.GetString("search.chat.model.name")
+		if key == "" {
+			return errors.ConfigValidationError("search.chat.model.name cannot be empty")
+		}
+		key = c.GetString("search.chat.model.vllm_url")
+		if key == "" {
+			return errors.ConfigValidationError("search.chat.model.vllm_url cannot be empty")
+		}
+		key = c.GetString("search.chat.model.max_bytes")
+		if key == "" {
+			return errors.ConfigValidationError("search.chat.model.max_bytes cannot be empty")
+		}
+	}
+
+
 	return nil
 }

backend/pkg/database/gorm_common.go

@@ -15,6 +15,7 @@ import (
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/event_bus"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/models"
 	databaseModel "github.com/fastenhealth/fasten-onprem/backend/pkg/models/database"
+	"github.com/fastenhealth/fasten-onprem/backend/pkg/search"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/utils"
 	sourceModel "github.com/fastenhealth/fasten-sources/clients/models"
 	"github.com/gin-gonic/gin"
@@ -367,8 +368,44 @@ func (gr *GormRepository) UpsertRawResource(ctx context.Context, sourceCredentia
 		}
 	}
 
-	return gr.UpsertResource(ctx, wrappedResourceModel)
+	success, err := gr.UpsertResource(ctx, wrappedResourceModel)
+	if err != nil {
+		return false, err
+	}
+
+	// Add nil checks before launching goroutine
+	if wrappedResourceModel == nil {
+		gr.Logger.Warn("Cannot index nil resource in Typesense")
+		return success, nil
+	}
+
+	if search.Client == nil {
+		gr.Logger.Warn("Search client is nil, skipping indexing")
+		return success, nil
+	}
 
+	indexer := &search.IndexerService{Client: search.Client}
+
+	// Launch goroutine with proper error handling
+	go func(resource *models.ResourceBase) {
+		// Additional safety check inside goroutine
+		if resource == nil {
+			gr.Logger.Warn("Resource is nil in indexing goroutine")
+			return
+		}
+
+		if indexer == nil {
+			gr.Logger.Warn("IndexerService is nil in indexing goroutine")
+			return
+		}
+
+		err := indexer.IndexResource(resource)
+		if err != nil {
+			gr.Logger.Warnf("Failed to index resource: %v", err)
+		}
+	}(wrappedResourceModel)
+
+	return success, nil
 }
 
 func (gr *GormRepository) UpsertRawResourceAssociation(
@@ -526,6 +563,50 @@ func (gr *GormRepository) ListResources(ctx context.Context, queryOptions models
 	}
 }
 
+// Used to query all resources across all tables and use them for Typesense indexing
+func (gr *GormRepository) ListAllResources(ctx context.Context, queryOptions models.ListResourceQueryOptions) ([]models.ResourceBase, error) {
+	queryParam := models.OriginBase{}
+
+	if len(queryOptions.SourceResourceType) > 0 {
+		queryParam.SourceResourceType = queryOptions.SourceResourceType
+	}
+	if len(queryOptions.SourceID) > 0 {
+		sourceUUID, err := uuid.Parse(queryOptions.SourceID)
+		if err != nil {
+			return nil, err
+		}
+		queryParam.SourceID = sourceUUID
+	}
+	if len(queryOptions.SourceResourceID) > 0 {
+		queryParam.SourceResourceID = queryOptions.SourceResourceID
+	}
+
+	queryBuilder := gr.GormClient.WithContext(ctx)
+	if len(queryOptions.SourceResourceType) > 0 {
+		tableName, err := databaseModel.GetTableNameByResourceType(queryOptions.SourceResourceType)
+		if err != nil {
+			return nil, err
+		}
+		queryBuilder = queryBuilder.
+			Where(queryParam).
+			Table(tableName)
+
+		if queryOptions.Limit > 0 {
+			queryBuilder = queryBuilder.Limit(queryOptions.Limit).Offset(queryOptions.Offset)
+		}
+
+		var wrappedResourceModels []models.ResourceBase
+		err = queryBuilder.Find(&wrappedResourceModels).Error
+		return wrappedResourceModels, err
+	}
+
+	// Query all resources across all tables
+	if queryOptions.Limit > 0 {
+		queryBuilder = queryBuilder.Limit(queryOptions.Limit).Offset(queryOptions.Offset)
+	}
+	return gr.getResourcesFromAllTables(queryBuilder, queryParam)
+}
+
 // TODO: should this be deprecated? (replaced by ListResources)
 func (gr *GormRepository) GetResourceByResourceTypeAndId(ctx context.Context, sourceResourceType string, sourceResourceId string) (*models.ResourceBase, error) {
 	currentUser, currentUserErr := gr.GetCurrentUser(ctx)

backend/pkg/database/gorm_repository_graph_test.go

@@ -3,6 +3,11 @@ package database
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"testing"
+
 	"github.com/fastenhealth/fasten-onprem/backend/pkg"
 	mock_config "github.com/fastenhealth/fasten-onprem/backend/pkg/config/mock"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/event_bus"
@@ -14,10 +19,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
-	"io/ioutil"
-	"log"
-	"os"
-	"testing"
 )
 
 // Define the suite, and absorb the built-in basic suite
@@ -61,6 +62,7 @@ func (suite *RepositoryGraphTestSuite) TestGetFlattenedResourceGraph() {
 	fakeConfig.EXPECT().GetString("database.location").Return(suite.TestDatabase.Name()).AnyTimes()
 	fakeConfig.EXPECT().GetString("database.type").Return("sqlite").AnyTimes()
 	fakeConfig.EXPECT().IsSet("database.encryption.key").Return(false).AnyTimes()
+	fakeConfig.EXPECT().IsSet("search").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetString("log.level").Return("INFO").AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.validation_mode").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.encryption.enabled").Return(false).AnyTimes()
@@ -131,6 +133,7 @@ func (suite *RepositoryGraphTestSuite) TestGetFlattenedResourceGraph_NDJson() {
 	fakeConfig.EXPECT().GetString("database.location").Return(suite.TestDatabase.Name()).AnyTimes()
 	fakeConfig.EXPECT().GetString("database.type").Return("sqlite").AnyTimes()
 	fakeConfig.EXPECT().IsSet("database.encryption.key").Return(false).AnyTimes()
+	fakeConfig.EXPECT().IsSet("search").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetString("log.level").Return("INFO").AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.validation_mode").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.encryption.enabled").Return(false).AnyTimes()

backend/pkg/database/gorm_repository_migrations.go

@@ -14,6 +14,7 @@ import (
 	_20250730100000 "github.com/fastenhealth/fasten-onprem/backend/pkg/database/migrations/20250730100000"
 	"github.com/fastenhealth/fasten-onprem/backend/pkg/models"
 	databaseModel "github.com/fastenhealth/fasten-onprem/backend/pkg/models/database"
+	"github.com/fastenhealth/fasten-onprem/backend/pkg/search"
 	sourceCatalog "github.com/fastenhealth/fasten-sources/catalog"
 	sourcePkg "github.com/fastenhealth/fasten-sources/pkg"
 	"github.com/go-gormigrate/gormigrate/v2"
@@ -267,6 +268,50 @@ func (gr *GormRepository) Migrate() error {
 		gr.Logger.Errorf("Final Database migration failed with error.\n Please open a github issue at https://github.com/fastenhealth/fasten-onprem. \n %v", err)
 	}
 
+	if gr.AppConfig.IsSet("search") && gr.AppConfig.GetString("search.uri") != "" {
+		indexer := &search.IndexerService{Client: search.Client}
+
+		// Index existing data if needed
+		ctx := context.Background()
+
+		systemSettings, err := gr.LoadSystemSettings(ctx)
+		if err != nil {
+			gr.Logger.Error("failed to load system settings: %w", err)
+		}
+
+		if systemSettings.TypesenseDataIndexed {
+			fmt.Print("Data already indexed, skipping...")
+		}
+
+		fmt.Print("Data not indexed. Indexing existing resources...")
+
+		if indexer.Client == nil {
+			gr.Logger.Error("Indexer client is nil!")
+		}
+
+		listResourceQueryOptions := models.ListResourceQueryOptions{}
+		resources, err := gr.ListAllResources(ctx, listResourceQueryOptions)
+		if err != nil {
+			gr.Logger.Error("failed to retrieve resources: %w", err)
+		}
+
+		for i := range resources {
+			if err := indexer.IndexResource(&resources[i]); err != nil {
+				gr.Logger.Error("Failed to index resource:", resources[i].ID, "-", err, "skipping...")
+				continue
+			}
+		}
+
+		systemSettings.TypesenseDataIndexed = true
+		if err := gr.SaveSystemSettings(ctx, systemSettings); err != nil {
+			gr.Logger.Error("failed to update system settings: %w", err)
+		}
+
+		gr.Logger.Print("Indexed %d resources", len(resources))
+		gr.Logger.Print("Indexing completed and flag updated.")
+	}
+
 	gr.Logger.Infoln("Database migration completed successfully")
+
 	return nil
 }

backend/pkg/database/gorm_repository_query_sql_test.go

@@ -45,6 +45,7 @@ func (suite *RepositorySqlTestSuite) BeforeTest(suiteName, testName string) {
 	fakeConfig.EXPECT().GetString("database.location").Return(suite.TestDatabase.Name()).AnyTimes()
 	fakeConfig.EXPECT().GetString("database.type").Return("sqlite").AnyTimes()
 	fakeConfig.EXPECT().IsSet("database.encryption.key").Return(false).AnyTimes()
+	fakeConfig.EXPECT().IsSet("search").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetString("log.level").Return("INFO").AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.validation_mode").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.encryption.enabled").Return(false).AnyTimes()

backend/pkg/database/gorm_repository_query_test.go

@@ -266,6 +266,7 @@ func (suite *RepositoryTestSuite) TestQueryResources_SQL() {
 	fakeConfig.EXPECT().GetString("database.location").Return(suite.TestDatabase.Name()).AnyTimes()
 	fakeConfig.EXPECT().GetString("database.type").Return("sqlite").AnyTimes()
 	fakeConfig.EXPECT().IsSet("database.encryption.key").Return(false).AnyTimes()
+	fakeConfig.EXPECT().IsSet("search").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetString("log.level").Return("INFO").AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.validation_mode").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.encryption.enabled").Return(false).AnyTimes()

backend/pkg/database/gorm_repository_related_test.go

@@ -41,6 +41,7 @@ func (suite *RepositoryRelatedTestSuite) SetupTest() {
 	fakeConfig.EXPECT().GetString("database.location").Return(suite.TestDatabase.Name()).AnyTimes()
 	fakeConfig.EXPECT().GetString("database.type").Return("sqlite").AnyTimes()
 	fakeConfig.EXPECT().IsSet("database.encryption.key").Return(false).AnyTimes()
+	fakeConfig.EXPECT().IsSet("search").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetString("log.level").Return("DEBUG").AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.validation_mode").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.encryption.enabled").Return(false).AnyTimes()

backend/pkg/database/gorm_repository_summary_test.go

@@ -62,6 +62,7 @@ func (suite *RepositorySummaryTestSuite) TestGetInternationalPatientSummaryBundl
 	fakeConfig.EXPECT().GetString("database.location").Return(suite.TestDatabase.Name()).AnyTimes()
 	fakeConfig.EXPECT().GetString("database.type").Return("sqlite").AnyTimes()
 	fakeConfig.EXPECT().IsSet("database.encryption.key").Return(false).AnyTimes()
+	fakeConfig.EXPECT().IsSet("search").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetString("log.level").Return("INFO").AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.validation_mode").Return(false).AnyTimes()
 	fakeConfig.EXPECT().GetBool("database.encryption.enabled").Return(false).AnyTimes()
@@ -99,7 +100,7 @@ func (suite *RepositorySummaryTestSuite) TestGetInternationalPatientSummaryBundl
 
 	manualClient, err := sourceFactory.GetSourceClient(sourcePkg.FastenLighthouseEnvSandbox, authContext, testLogger, &testSourceCredential)
 	require.NoError(suite.T(), err)
-	
+
 	summary, err := manualClient.SyncAllBundle(dbRepo, bundleFile, sourcePkg.FhirVersion401)
 	require.NoError(suite.T(), err)
 	require.Equal(suite.T(), 72, summary.TotalResources)