Skip to content

Commit b611618

Browse files
loverustfscursoragentcxymds
authored
feat(transfer): align cross-alias copy, move, and diff on source identity (#354)
Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: cxymds <cxymds@gmail.com> Co-authored-by: 马登山 <cxymds@qq.com>
1 parent 9d91bde commit b611618

14 files changed

Lines changed: 1517 additions & 199 deletions

File tree

‎crates/cli/src/commands/cp.rs‎

Lines changed: 143 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ use crate::exit_code::ExitCode;
2525
use crate::output::{Formatter, OutputConfig, ProgressBar, V3SuccessEnvelope};
2626
use crate::secret_input::{SecretLocator, resolve_secret_locator};
2727

28+
use super::object_identity::set_source_identity;
2829
use super::transfer_fidelity::{MetadataDirectiveArg, TaggingDirectiveArg, TransferFidelityArgs};
2930

3031
const CP_AFTER_HELP: &str = "\
@@ -689,7 +690,10 @@ fn validate_fidelity_directions(
689690
&& sources.iter().any(|source| {
690691
matches!(
691692
source,
692-
ParsedPath::Remote(source) if target.as_remote().is_some_and(|target| source.alias == target.alias)
693+
ParsedPath::Remote(source)
694+
if target
695+
.as_remote()
696+
.is_some_and(|target| source.alias == target.alias)
693697
)
694698
});
695699
if any_remote && target_remote {
@@ -1245,6 +1249,7 @@ async fn perform_planned_download(
12451249
struct PlannedRemoteCopyResult {
12461250
bytes_copied: u64,
12471251
source_version_id: Option<String>,
1252+
source_etag: Option<String>,
12481253
destination_version_id: Option<String>,
12491254
upload_id: Option<String>,
12501255
object: ObjectInfo,
@@ -1312,6 +1317,7 @@ async fn perform_planned_remote_copy(
13121317
return Ok(PlannedRemoteCopyResult {
13131318
bytes_copied: copied.bytes_copied,
13141319
source_version_id: options.source_version_id,
1320+
source_etag: current.etag.clone(),
13151321
destination_version_id: copied.object.version_id.clone(),
13161322
upload_id: Some(copied.upload_id),
13171323
object: copied.object,
@@ -1332,6 +1338,7 @@ async fn perform_planned_remote_copy(
13321338
.source_version_id
13331339
.clone()
13341340
.or_else(|| source_info.version_id.clone()),
1341+
source_etag: source_info.etag.clone(),
13351342
destination_version_id: copied.version_id.clone(),
13361343
upload_id: None,
13371344
object: copied,
@@ -1427,6 +1434,7 @@ async fn perform_cross_alias_remote_copy(
14271434
Ok(PlannedRemoteCopyResult {
14281435
bytes_copied,
14291436
source_version_id: current.version_id.clone(),
1437+
source_etag: current.etag.clone(),
14301438
destination_version_id: object.version_id.clone(),
14311439
upload_id: None,
14321440
object,
@@ -1451,6 +1459,46 @@ fn source_identity_matches(planned: &ObjectInfo, current: &ObjectInfo) -> bool {
14511459
}
14521460
}
14531461

1462+
/// Copy one object between two aliases by streaming through the client.
1463+
///
1464+
/// `rc mv` shares this path so a cross-alias move behaves exactly like a
1465+
/// cross-alias copy followed by a source delete, rather than reimplementing the
1466+
/// download/upload streaming and its source-change checks.
1467+
#[derive(Debug, Clone)]
1468+
pub(super) struct CrossAliasCopyResult {
1469+
pub(super) object: ObjectInfo,
1470+
pub(super) source_version_id: Option<String>,
1471+
pub(super) source_etag: Option<String>,
1472+
}
1473+
1474+
pub(super) async fn copy_object_across_aliases(
1475+
source_client: &S3Client,
1476+
target_client: &S3Client,
1477+
source: &RemotePath,
1478+
target: &RemotePath,
1479+
encryption: Option<&ObjectEncryptionRequest>,
1480+
) -> rc_core::Result<CrossAliasCopyResult> {
1481+
let source_info = source_client.head_object(source).await?;
1482+
let args = CpArgs::single(source.to_string(), target.to_string());
1483+
let ignore_progress = |_: u64| {};
1484+
let result = perform_cross_alias_remote_copy(
1485+
source_client,
1486+
target_client,
1487+
source,
1488+
target,
1489+
&source_info,
1490+
encryption,
1491+
&ignore_progress,
1492+
&args,
1493+
)
1494+
.await?;
1495+
Ok(CrossAliasCopyResult {
1496+
object: result.object,
1497+
source_version_id: result.source_version_id,
1498+
source_etag: result.source_etag,
1499+
})
1500+
}
1501+
14541502
fn piped_copy_write_options(
14551503
args: &CpArgs,
14561504
source: &ObjectInfo,
@@ -1463,20 +1511,27 @@ fn piped_copy_write_options(
14631511
args.destination_customer_key.as_ref(),
14641512
args.storage_class.clone(),
14651513
)?;
1466-
if matches!(
1514+
let replace_metadata = matches!(
14671515
requested_metadata_directive(args),
14681516
Some(MetadataDirective::Replace)
1469-
) {
1470-
return Ok(options);
1471-
}
1517+
);
14721518
let mut attributes = options.attributes.take().unwrap_or_default();
1473-
if attributes.content_type.is_none() {
1474-
attributes.content_type = source.content_type.clone();
1519+
if !replace_metadata {
1520+
if attributes.content_type.is_none() {
1521+
attributes.content_type = source.content_type.clone();
1522+
}
1523+
if attributes.user_metadata.is_empty()
1524+
&& let Some(metadata) = &source.metadata
1525+
{
1526+
attributes.user_metadata.clone_from(metadata);
1527+
}
14751528
}
1476-
if attributes.user_metadata.is_empty()
1477-
&& let Some(metadata) = &source.metadata
1478-
{
1479-
attributes.user_metadata.clone_from(metadata);
1529+
// The destination computes its own ETag, so record the source ETag the same
1530+
// way `rc mirror` does. Without this a later `mirror --compare auto` cannot
1531+
// tell a faithful cross-alias copy from a changed object and recopies it.
1532+
// This is `rc` bookkeeping rather than user data, so it survives --metadata-directive replace.
1533+
if let Some(source_etag) = source.etag.as_deref() {
1534+
set_source_identity(&mut attributes, source_etag);
14801535
}
14811536
if attributes != ObjectAttributes::default() {
14821537
options.attributes = Some(attributes);
@@ -1502,7 +1557,14 @@ fn multipart_options_from_source(source: &ObjectInfo) -> rc_core::Result<Multipa
15021557
let mut options = MultipartCopyOptions::new(source_size, source_etag)?;
15031558
options.source_version_id = source.version_id.clone();
15041559
options.content_type = source.content_type.clone();
1505-
options.metadata = source.metadata.clone().unwrap_or_default();
1560+
let mut attributes = ObjectAttributes {
1561+
user_metadata: source.metadata.clone().unwrap_or_default(),
1562+
..ObjectAttributes::default()
1563+
};
1564+
if let Some(source_etag) = source.etag.as_deref() {
1565+
set_source_identity(&mut attributes, source_etag);
1566+
}
1567+
options.metadata = attributes.user_metadata;
15061568
Ok(options)
15071569
}
15081570

@@ -3915,6 +3977,10 @@ mod tests {
39153977
options.metadata.get("project").map(String::as_str),
39163978
Some("archive")
39173979
);
3980+
assert_eq!(
3981+
options.metadata.get("rc-source-etag").map(String::as_str),
3982+
Some("planned-etag")
3983+
);
39183984
}
39193985

39203986
#[test]
@@ -4124,7 +4190,7 @@ mod tests {
41244190
&cross_alias_target,
41254191
)
41264192
.is_ok(),
4127-
"metadata REPLACE is implemented by the cross-alias upload path"
4193+
"cross-alias metadata REPLACE uses the upload path"
41284194
);
41294195

41304196
let mut tags = CpArgs::single("test/source/report.json", "test/target/report.json");
@@ -4144,15 +4210,14 @@ mod tests {
41444210
}
41454211

41464212
#[test]
4147-
fn source_identity_validation_detects_same_size_etag_changes() {
4213+
fn source_identity_validation_rejects_same_size_etag_changes() {
41484214
let mut planned = ObjectInfo::file("report.json", 4);
41494215
planned.etag = Some("planned".to_string());
41504216
let mut current = planned.clone();
41514217
assert!(source_identity_matches(&planned, &current));
41524218

41534219
current.etag = Some("changed".to_string());
41544220
assert!(!source_identity_matches(&planned, &current));
4155-
41564221
current.etag = None;
41574222
assert!(!source_identity_matches(&planned, &current));
41584223

@@ -4262,6 +4327,69 @@ mod tests {
42624327
);
42634328
}
42644329

4330+
#[test]
4331+
fn piped_copy_records_the_source_etag_as_identity_metadata() {
4332+
let mut source = ObjectInfo::file("file.txt", 4);
4333+
source.etag = Some("source-etag".to_string());
4334+
let args = CpArgs::single("alpha/source/file.txt", "beta/target/file.txt");
4335+
4336+
let options = piped_copy_write_options(&args, &source, None).expect("copy metadata");
4337+
4338+
let attributes = options.attributes.as_ref().expect("identity attributes");
4339+
assert_eq!(
4340+
super::super::object_identity::identity_etag_from_metadata(Some(
4341+
&attributes.user_metadata
4342+
))
4343+
.as_deref(),
4344+
Some("source-etag"),
4345+
"a later mirror --compare auto must be able to skip this object"
4346+
);
4347+
}
4348+
4349+
#[test]
4350+
fn piped_copy_records_identity_even_when_metadata_is_replaced() {
4351+
let mut source = ObjectInfo::file("file.txt", 4);
4352+
source.etag = Some("source-etag".to_string());
4353+
source.metadata = Some(HashMap::from([(
4354+
"owner".to_string(),
4355+
"storage".to_string(),
4356+
)]));
4357+
let mut args = CpArgs::single("alpha/source/file.txt", "beta/target/file.txt");
4358+
args.metadata_directive = Some(MetadataDirectiveArg::Replace);
4359+
4360+
let options = piped_copy_write_options(&args, &source, None).expect("replace metadata");
4361+
4362+
let attributes = options.attributes.as_ref().expect("identity attributes");
4363+
assert_eq!(
4364+
super::super::object_identity::identity_etag_from_metadata(Some(
4365+
&attributes.user_metadata
4366+
))
4367+
.as_deref(),
4368+
Some("source-etag"),
4369+
"identity is rc bookkeeping, not user metadata"
4370+
);
4371+
assert!(
4372+
!attributes.user_metadata.contains_key("owner"),
4373+
"replace must still drop source user metadata"
4374+
);
4375+
}
4376+
4377+
#[test]
4378+
fn piped_copy_omits_identity_when_the_source_has_no_etag() {
4379+
let source = ObjectInfo::file("file.txt", 4);
4380+
let args = CpArgs::single("alpha/source/file.txt", "beta/target/file.txt");
4381+
4382+
let options = piped_copy_write_options(&args, &source, None).expect("copy metadata");
4383+
4384+
assert!(
4385+
options
4386+
.attributes
4387+
.as_ref()
4388+
.is_none_or(|attributes| attributes.user_metadata.is_empty()),
4389+
"without a source ETag there is no identity to record"
4390+
);
4391+
}
4392+
42654393
#[test]
42664394
fn piped_copy_preserves_source_user_metadata_unless_replaced() {
42674395
let mut source = ObjectInfo::file("file.txt", 4);

0 commit comments

Comments
 (0)