diff --git a/app/(dashboard)/on-demand-migration/page.tsx b/app/(dashboard)/on-demand-migration/page.tsx new file mode 100644 index 00000000..a136cffa --- /dev/null +++ b/app/(dashboard)/on-demand-migration/page.tsx @@ -0,0 +1,64 @@ +"use client" + +import Link from "next/link" +import { useSearchParams } from "next/navigation" +import { useTranslation } from "react-i18next" +import { RiArrowLeftLine } from "@remixicon/react" +import { BucketList } from "@/components/buckets/list" +import { OnDemandMigrationManagement } from "@/components/on-demand-migration/management" +import { Page } from "@/components/page" +import { PageHeader } from "@/components/page-header" +import { Button } from "@/components/ui/button" +import { buildModuleBucketPath } from "@/lib/module-bucket-route" + +export default function OnDemandMigrationPage() { + const { t } = useTranslation() + const searchParams = useSearchParams() + const bucketName = searchParams.get("bucket") ?? "" + + if (!bucketName) { + return ( + + {t("On-demand migration")}} + emptyDescription={t("Create a bucket before configuring an external migration source.")} + getBucketHref={(name) => buildModuleBucketPath("/on-demand-migration", name)} + /> + + ) + } + + return ( + + ( + + {t("Bucket")}: {bucketName} +

+ } + actions={ + <> + + {actions} + + } + > +

{t("On-demand migration")}

+ + )} + /> + + ) +} diff --git a/components/buckets/info.tsx b/components/buckets/info.tsx index 26972780..b70cc059 100644 --- a/components/buckets/info.tsx +++ b/components/buckets/info.tsx @@ -8,6 +8,7 @@ import { useBucket } from "@/hooks/use-bucket" import { usePermissions } from "@/hooks/use-permissions" import { useSSE } from "@/hooks/use-sse" import { BucketSettingRow, BucketSettingsSection } from "@/components/buckets/settings-layout" +import { OnDemandMigrationSettingsRow } from "@/components/on-demand-migration/settings-row" import { Button } from "@/components/ui/button" import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert" import { Switch } from "@/components/ui/switch" @@ -1061,6 +1062,7 @@ export function BucketInfo({ bucketName }: BucketInfoProps) { } /> ))} + diff --git a/components/on-demand-migration/backfill-dialog.tsx b/components/on-demand-migration/backfill-dialog.tsx new file mode 100644 index 00000000..222b3865 --- /dev/null +++ b/components/on-demand-migration/backfill-dialog.tsx @@ -0,0 +1,331 @@ +"use client" + +import * as React from "react" +import Link from "next/link" +import { useTranslation } from "react-i18next" +import { RiErrorWarningLine, RiPlayLine, RiStopCircleLine } from "@remixicon/react" +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert" +import { Button } from "@/components/ui/button" +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { Field, FieldContent, FieldDescription, FieldGroup, FieldLabel } from "@/components/ui/field" +import { Input } from "@/components/ui/input" +import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" +import { Spinner } from "@/components/ui/spinner" +import { Switch } from "@/components/ui/switch" +import { useOnDemandMigration } from "@/hooks/use-on-demand-migration" +import { useDialog } from "@/lib/feedback/dialog" +import { useMessage } from "@/lib/feedback/message" +import { formatBytes, formatInteger } from "@/lib/functions" +import { getOnDemandMigrationErrorKind, isOnDemandMigrationBackfillActive } from "@/lib/on-demand-migration" +import { scheduleMicrotask } from "@/lib/schedule-microtask" +import type { OnDemandMigrationBackfillJob, OnDemandMigrationSkipExisting } from "@/types/on-demand-migration" + +interface BackfillDialogProps { + open: boolean + onOpenChange: (open: boolean) => void + bucketName: string + moduleEnabled: boolean + job: OnDemandMigrationBackfillJob | null + onChanged: () => void | Promise +} + +function errorMessage(error: unknown): string { + return error instanceof Error && error.message ? error.message : String(error) +} + +export function OnDemandMigrationBackfillDialog({ + open, + onOpenChange, + bucketName, + moduleEnabled, + job, + onChanged, +}: BackfillDialogProps) { + const { t } = useTranslation() + const message = useMessage() + const dialog = useDialog() + const { startBackfill, cancelBackfill } = useOnDemandMigration() + const [prefix, setPrefix] = React.useState("") + const [skipExisting, setSkipExisting] = React.useState("always") + const [dryRun, setDryRun] = React.useState(false) + const [operationError, setOperationError] = React.useState(null) + const [submitting, setSubmitting] = React.useState<"start" | "cancel" | null>(null) + const active = isOnDemandMigrationBackfillActive(job?.state) + + React.useEffect(() => { + if (!open) return + scheduleMicrotask(() => { + setPrefix("") + setSkipExisting("always") + setDryRun(false) + setOperationError(null) + setSubmitting(null) + }) + }, [open]) + + const handleStart = async () => { + if (!moduleEnabled || active || submitting) return + setSubmitting("start") + setOperationError(null) + try { + await startBackfill(bucketName, { + ...(prefix ? { prefix } : {}), + skip_existing: skipExisting, + dry_run: dryRun, + }) + message.success(dryRun ? t("Backfill dry run started") : t("Backfill started")) + await onChanged() + onOpenChange(false) + } catch (error) { + setOperationError(error) + } finally { + setSubmitting(null) + } + } + + const performCancel = async () => { + if (!active || submitting) return + setSubmitting("cancel") + setOperationError(null) + try { + await cancelBackfill(bucketName) + message.success(t("Backfill cancellation requested")) + await onChanged() + onOpenChange(false) + } catch (error) { + setOperationError(error) + } finally { + setSubmitting(null) + } + } + + const confirmCancel = () => { + dialog.warning({ + title: t("Cancel the running backfill?"), + content: t("Objects already migrated remain in the local bucket. You can start a new job later."), + positiveText: t("Cancel backfill"), + negativeText: t("Keep running"), + onPositiveClick: performCancel, + }) + } + + const errorKind = getOnDemandMigrationErrorKind(operationError) + + return ( + { + if (nextOpen) onOpenChange(true) + else if (!submitting) onOpenChange(false) + }} + disablePointerDismissal + > + + + {active ? t("Backfill in progress") : t("Start background backfill")} + + {t("Bucket")}: {bucketName} + + + +
+ {operationError ? ( + + + + {errorKind === "backfill_running" + ? t("A backfill job is already running") + : errorKind === "module_disabled" + ? t("On-demand migration is not enabled on this server") + : errorKind === "license_denied" + ? t("The server rejected this operation") + : errorKind === "access_denied" + ? t("You do not have permission to control backfill") + : t("Backfill operation failed")} + + +

{errorMessage(operationError)}

+ {errorKind === "license_denied" ? ( +

+ {t("Open license settings")} +

+ ) : null} + + + ) : null} + + {!moduleEnabled ? ( + + + {t("On-demand migration is not enabled on this server")} + + {t("Backfill cannot start until the module is enabled on every server node.")} + + + ) : null} + + {active && job ? ( +
+

+ {t("The source is still being scanned, so no completion percentage is shown.")} +

+
+
+
{t("Objects listed")}
+
{formatInteger(job.listed)}
+
+
+
{t("Objects pulled")}
+
{formatInteger(job.pulled)}
+
+
+
+
+
{t("Skipped existing")}
+
{formatInteger(job.skipped_existing)}
+
+
+
{t("Failed objects")}
+
0 ? "text-destructive tabular-nums" : "tabular-nums"}> + {formatInteger(job.failed)} +
+
+
+
{t("Migrated data")}
+
{formatBytes(job.bytes)}
+
+
+
{t("Job ID")}
+
{job.job_id}
+
+
+
+ ) : ( + + + {t("Source listing prefix (optional)")} + + { + setPrefix(event.target.value) + setOperationError(null) + }} + /> + + {t("Leave empty to scan the complete configured source namespace.")} + + + + + + {t("Existing local objects")} + + + + {skipExisting === "always" + ? t("Any current local object is skipped.") + : t("Objects whose recorded source ETag and size differ are pulled again.")} + + + + + + + {t("Dry Run")} + + {t("List matching objects without queueing or writing any object.")} + + + + + + )} +
+ + + + {active ? ( + + ) : ( + + )} + + + + ) +} diff --git a/components/on-demand-migration/config-dialog.tsx b/components/on-demand-migration/config-dialog.tsx new file mode 100644 index 00000000..04afedf7 --- /dev/null +++ b/components/on-demand-migration/config-dialog.tsx @@ -0,0 +1,957 @@ +"use client" + +import * as React from "react" +import Link from "next/link" +import { useTranslation } from "react-i18next" +import { + RiArrowDownSLine, + RiCheckLine, + RiErrorWarningLine, + RiFlaskLine, + RiInformationLine, + RiSaveLine, +} from "@remixicon/react" +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert" +import { Button } from "@/components/ui/button" +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible" +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { + Field, + FieldContent, + FieldDescription, + FieldError, + FieldGroup, + FieldLabel, + FieldLegend, + FieldSet, +} from "@/components/ui/field" +import { Input } from "@/components/ui/input" +import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" +import { Spinner } from "@/components/ui/spinner" +import { Switch } from "@/components/ui/switch" +import { Textarea } from "@/components/ui/textarea" +import { useOnDemandMigration } from "@/hooks/use-on-demand-migration" +import { useDialog } from "@/lib/feedback/dialog" +import { useMessage } from "@/lib/feedback/message" +import { + buildOnDemandMigrationConfig, + createOnDemandMigrationFormValues, + getOnDemandMigrationErrorKind, + getOnDemandMigrationServerField, + ODM_AUTO_REGION_PROVIDERS, + ODM_NATIVE_PROVIDERS, + ODM_OPTIONAL_ENDPOINT_PROVIDERS, + ON_DEMAND_MIGRATION_PROVIDERS, + validateOnDemandMigrationForm, +} from "@/lib/on-demand-migration" +import { scheduleMicrotask } from "@/lib/schedule-microtask" +import type { + OnDemandMigrationConfig, + OnDemandMigrationFormField, + OnDemandMigrationFormValues, + OnDemandMigrationProbe, + OnDemandMigrationProvider, +} from "@/types/on-demand-migration" + +interface ConfigDialogProps { + open: boolean + onOpenChange: (open: boolean) => void + bucketName: string + config: OnDemandMigrationConfig | null + moduleEnabled: boolean + backfillActive: boolean + onSaved: () => void | Promise +} + +type FormErrors = Partial> + +const ADVANCED_FIELDS = new Set([ + "negativeCacheTtlSecs", + "inlineMaxBytes", + "multipartPartSizeBytes", + "maxConcurrentPulls", + "pullQueueCapacity", + "connectTimeoutMs", + "firstByteTimeoutMs", + "idleTimeoutMs", + "bandwidthLimitBytesPerSec", + "caCertPem", +]) + +const FIELD_IDS: Record = { + enabled: "odm-enabled", + provider: "odm-provider", + endpoint: "odm-endpoint", + region: "odm-region", + bucket: "odm-source-bucket", + pathStyle: "odm-path-style", + accessKey: "odm-access-key", + secretKey: "odm-secret-key", + sessionToken: "odm-session-token", + azureAccount: "odm-azure-account", + azureAuth: "odm-azure-auth", + azureSecret: "odm-azure-secret", + serviceAccountJson: "odm-service-account-json", + localPrefix: "odm-local-prefix", + sourcePrefix: "odm-source-prefix", + skipTlsVerify: "odm-skip-tls-verify", + caCertPem: "odm-ca-cert-pem", + head: "odm-head-policy", + rangeGet: "odm-range-get-policy", + sourceError: "odm-source-error-policy", + listThrough: "odm-list-through", + respectLocalDeleteMarker: "odm-respect-delete-marker", + preserveEtag: "odm-preserve-etag", + copyTags: "odm-copy-tags", + emitEvents: "odm-emit-events", + negativeCacheTtlSecs: "odm-negative-cache-ttl", + inlineMaxBytes: "odm-inline-max", + multipartPartSizeBytes: "odm-multipart-part-size", + maxConcurrentPulls: "odm-max-concurrent-pulls", + pullQueueCapacity: "odm-pull-queue-capacity", + connectTimeoutMs: "odm-connect-timeout", + firstByteTimeoutMs: "odm-first-byte-timeout", + idleTimeoutMs: "odm-idle-timeout", + bandwidthLimitBytesPerSec: "odm-bandwidth-limit", +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error && error.message ? error.message : String(error) +} + +export function OnDemandMigrationConfigDialog({ + open, + onOpenChange, + bucketName, + config, + moduleEnabled, + backfillActive, + onSaved, +}: ConfigDialogProps) { + const { t } = useTranslation() + const message = useMessage() + const dialog = useDialog() + const { setConfig } = useOnDemandMigration() + const [values, setValues] = React.useState(() => + createOnDemandMigrationFormValues(config ?? undefined), + ) + const [initialFingerprint, setInitialFingerprint] = React.useState("") + const [errors, setErrors] = React.useState({}) + const [operationError, setOperationError] = React.useState(null) + const [probe, setProbe] = React.useState(null) + const [testing, setTesting] = React.useState(false) + const [saving, setSaving] = React.useState(false) + const [advancedOpen, setAdvancedOpen] = React.useState(false) + const wasOpenRef = React.useRef(false) + + const fingerprint = React.useMemo(() => JSON.stringify(values), [values]) + const dirty = initialFingerprint !== "" && fingerprint !== initialFingerprint + const nativeProvider = ODM_NATIVE_PROVIDERS.has(values.provider) + const endpointOptional = ODM_OPTIONAL_ENDPOINT_PROVIDERS.has(values.provider) + const editing = config !== null + + React.useEffect(() => { + if (!open) { + wasOpenRef.current = false + return + } + if (wasOpenRef.current) return + wasOpenRef.current = true + scheduleMicrotask(() => { + const nextValues = createOnDemandMigrationFormValues(config ?? undefined) + setValues(nextValues) + setInitialFingerprint(JSON.stringify(nextValues)) + setErrors({}) + setOperationError(null) + setProbe(null) + setTesting(false) + setSaving(false) + setAdvancedOpen(false) + }) + }, [config, open]) + + const providerItems = React.useMemo( + () => + ON_DEMAND_MIGRATION_PROVIDERS.map((value) => ({ + value, + label: + value === "aws" + ? t("Amazon S3") + : value === "s3" + ? t("S3-compatible storage") + : value === "minio" + ? "MinIO" + : value === "rustfs" + ? "RustFS" + : value === "r2" + ? "Cloudflare R2" + : value === "gcs" + ? t("Google Cloud Storage (HMAC)") + : value === "azure" + ? t("Azure Blob Storage (Native)") + : t("Google Cloud Storage (Native)"), + })), + [t], + ) + const providerLabel = providerItems.find((item) => item.value === values.provider)?.label ?? values.provider + + const update = React.useCallback( + (field: K, value: OnDemandMigrationFormValues[K]) => { + setValues((current) => ({ ...current, [field]: value })) + setErrors((current) => ({ ...current, [field]: undefined })) + setOperationError(null) + setProbe(null) + }, + [], + ) + + const updateProvider = (provider: OnDemandMigrationProvider) => { + setValues((current) => { + const native = ODM_NATIVE_PROVIDERS.has(provider) + const region = native + ? "auto" + : current.region === "auto" && !ODM_AUTO_REGION_PROVIDERS.has(provider) + ? "us-east-1" + : current.region || (ODM_AUTO_REGION_PROVIDERS.has(provider) ? "auto" : "us-east-1") + return { ...current, provider, region, pathStyle: native ? "auto" : current.pathStyle } + }) + setErrors({}) + setOperationError(null) + setProbe(null) + } + + const validate = () => { + const issues = validateOnDemandMigrationForm(values) + const nextErrors: FormErrors = {} + for (const issue of issues) nextErrors[issue.field] ??= t(issue.message) + setErrors(nextErrors) + if (issues[0]) { + if (ADVANCED_FIELDS.has(issues[0].field)) { + setAdvancedOpen(true) + } + scheduleMicrotask(() => document.getElementById(FIELD_IDS[issues[0].field])?.focus()) + } + return issues.length === 0 + } + + const handleError = (error: unknown) => { + setOperationError(error) + const serverField = getOnDemandMigrationServerField(getErrorMessage(error)) + if (serverField) { + setErrors((current) => ({ ...current, [serverField]: getErrorMessage(error) })) + if (ADVANCED_FIELDS.has(serverField)) { + setAdvancedOpen(true) + } + scheduleMicrotask(() => document.getElementById(FIELD_IDS[serverField])?.focus()) + } + } + + const handleTest = async () => { + if (!moduleEnabled || testing || saving || !validate()) return + setTesting(true) + setOperationError(null) + setProbe(null) + try { + const response = await setConfig(bucketName, buildOnDemandMigrationConfig(values), true) + setProbe(response.probe) + message.success(t("Connection test passed")) + } catch (error) { + handleError(error) + } finally { + setTesting(false) + } + } + + const handleSave = async () => { + if (!moduleEnabled || testing || saving || !validate()) return + setSaving(true) + setOperationError(null) + try { + await setConfig(bucketName, buildOnDemandMigrationConfig(values), false) + message.success(editing ? t("Migration configuration updated") : t("On-demand migration enabled")) + setInitialFingerprint("") + onOpenChange(false) + await onSaved() + } catch (error) { + handleError(error) + } finally { + setSaving(false) + } + } + + const closeNow = React.useCallback(() => { + setInitialFingerprint("") + onOpenChange(false) + }, [onOpenChange]) + + const requestClose = React.useCallback(() => { + if (testing || saving) return + if (!dirty) { + closeNow() + return + } + dialog.warning({ + title: t("Discard unsaved changes?"), + content: t("Your migration configuration changes will be lost."), + positiveText: t("Discard changes"), + negativeText: t("Keep editing"), + onPositiveClick: closeNow, + }) + }, [closeNow, dialog, dirty, saving, t, testing]) + + const renderTextField = ( + field: OnDemandMigrationFormField, + label: string, + description: string, + options: React.ComponentProps = {}, + ) => { + const id = FIELD_IDS[field] + const error = errors[field] + const value = values[field] + return ( + + {label} + + update(field, event.target.value as never)} + /> + {description} + {error} + + + ) + } + + const renderSwitch = (field: OnDemandMigrationFormField, label: string, description: string, disabled = false) => { + const id = FIELD_IDS[field] + return ( + + + {label} + {description} + + update(field, checked as never)} + /> + + ) + } + + const errorKind = getOnDemandMigrationErrorKind(operationError) + const errorMessage = operationError ? getErrorMessage(operationError) : "" + + return ( + { + if (nextOpen) onOpenChange(true) + else requestClose() + }} + disablePointerDismissal + > + + + {editing ? t("Edit on-demand migration") : t("Enable on-demand migration")} + + {t("Bucket")}: {bucketName} + + + +
+ {!moduleEnabled ? ( + + + {t("On-demand migration is not enabled on this server")} + + {t("Enable RUSTFS_ON_DEMAND_MIGRATION_ENABLED on every server node, restart, and try again.")} + + + ) : null} + + {editing ? ( + + + {t("Credentials are write-only")} + + {t("For security, re-enter the source credential before testing or saving any change.")} + + + ) : null} + + {backfillActive ? ( + + + {t("Editing cancels the active backfill")} + + {t( + "Saving a replacement configuration cancels the current job. You can start a new backfill afterward.", + )} + + + ) : null} + + {operationError ? ( + + + + {errorKind === "module_disabled" + ? t("On-demand migration is not enabled on this server") + : errorKind === "source_unreachable" + ? t("The source could not be reached") + : errorKind === "backend_not_compiled" + ? t("This backend is not included in the server build") + : errorKind === "license_denied" + ? t("The server rejected this operation") + : errorKind === "access_denied" + ? t("You do not have permission to change migration settings") + : t("Migration configuration could not be saved")} + + +

{errorMessage}

+ {errorKind === "license_denied" ? ( +

+ {t("Open license settings")} +

+ ) : null} + + + ) : null} + + {probe ? ( + + + {t("Connection test passed")} + +

{t("RustFS reached the source bucket and listed it successfully.")}

+ {probe.sample_key ? ( +

+ {t("Sample object")}: {probe.sample_key} +

+ ) : ( +

{t("The source bucket is empty or the selected prefix has no objects.")}

+ )} + + + ) : null} + + +
+ {t("Source connection")} + + + {t("Provider")} + + + + {values.provider === "gcs" + ? t("Uses the S3 interoperability API and an HMAC key pair.") + : values.provider === "gcs_native" + ? t("Uses the native GCS API and a service-account JSON key.") + : values.provider === "azure" + ? t("Uses the native Azure Blob API.") + : t("RustFS reads from this source and never writes or deletes objects there.")} + + + + + {renderTextField( + "endpoint", + endpointOptional ? t("Custom endpoint (optional)") : t("Endpoint"), + values.provider === "aws" + ? t("Leave empty to use the regional Amazon S3 endpoint.") + : values.provider === "azure" + ? t("Leave empty to use the account's public Azure Blob endpoint.") + : values.provider === "gcs_native" + ? t("Leave empty to use https://storage.googleapis.com.") + : t("Use an absolute HTTP or HTTPS URL with no path."), + { + type: "url", + placeholder: endpointOptional ? t("Use provider default") : "https://source.example.com", + }, + )} + + {!nativeProvider + ? renderTextField( + "region", + t("Region"), + ODM_AUTO_REGION_PROVIDERS.has(values.provider) + ? t("Enter a real region or auto.") + : t("This provider requires a real region; auto is not supported."), + { autoComplete: "off", placeholder: "us-east-1" }, + ) + : null} + + {renderTextField( + "bucket", + values.provider === "azure" ? t("Source container") : t("Source Bucket"), + t("The existing source that RustFS will read from."), + { autoComplete: "off" }, + )} + + {!nativeProvider ? ( + + {t("Addressing style")} + + + + {t("Use an explicit style when the source redirects or reports a missing bucket.")} + + + + ) : null} + + {values.provider === "azure" ? ( + <> + {renderTextField( + "azureAccount", + t("Azure storage account"), + t("Used to derive the public endpoint when no custom endpoint is set."), + { autoComplete: "off" }, + )} + + {t("Azure credential type")} + + + + + {renderTextField( + "azureSecret", + values.azureAuth === "account_key" ? t("Account key") : t("SAS token"), + values.azureAuth === "account_key" + ? t("Enter the Base64 storage account key.") + : t("Enter the SAS query string without a leading question mark."), + { type: "password", autoComplete: "new-password" }, + )} + + ) : values.provider === "gcs_native" ? ( + + {t("Service-account JSON key")} + +