fix: omit filesize in symlinks

packages/ipfs-unixfs-importer/src/dag-builder/index.ts

@@ -2,11 +2,13 @@ import { CustomProgressEvent } from 'progress-events'
 import { InvalidContentError } from '../errors.ts'
 import { defaultDirBuilder } from './dir.ts'
 import { defaultFileBuilder } from './file.ts'
+import { defaultSymlinkBuilder } from './symlink.ts'
 import type { DirBuilder, DirBuilderOptions } from './dir.ts'
 import type { FileBuilder, FileBuilderOptions } from './file.ts'
+import type { SymlinkBuilder } from './symlink.ts'
 import type { ChunkValidator } from './validate-chunks.ts'
 import type { Chunker } from '../chunker/index.ts'
-import type { Directory, File, FileCandidate, ImportCandidate, ImporterProgressEvents, InProgressImportResult, WritableStorage } from '../index.ts'
+import type { Directory, File, FileCandidate, ImportCandidate, ImporterProgressEvents, InProgressImportResult, Symlink, SymlinkCandidate, WritableStorage } from '../index.ts'
 import type { ProgressEvent, ProgressOptions } from 'progress-events'
 
 /**
@@ -66,6 +68,7 @@ export interface DagBuilderOptions extends FileBuilderOptions, DirBuilderOptions
   wrapWithDirectory: boolean
   dirBuilder?: DirBuilder
   fileBuilder?: FileBuilder
+  symlinkBuilder?: SymlinkBuilder
 }
 
 export type ImporterSourceStream = AsyncIterable<ImportCandidate> | Iterable<ImportCandidate>
@@ -74,12 +77,16 @@ export interface DAGBuilder {
   (source: ImporterSourceStream, blockstore: WritableStorage): AsyncIterable<() => Promise<InProgressImportResult>>
 }
 
+function isPathCandidate (candidate: any): candidate is { path: string } {
+  return typeof candidate.path === 'string'
+}
+
 export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder {
   return async function * dagBuilder (source, blockstore) {
     for await (const entry of source) {
       let originalPath: string | undefined
 
-      if (entry.path != null) {
+      if (isPathCandidate(entry)) {
         originalPath = entry.path
         entry.path = entry.path
           .split('/')
@@ -114,6 +121,18 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder {
         const fileBuilder = options.fileBuilder ?? defaultFileBuilder
 
         yield async () => fileBuilder(file, blockstore, options)
+      } else if (isSymLinkCandidate(entry)) {
+        const symlink: Symlink = {
+          path: entry.path,
+          link: entry.link,
+          mtime: entry.mtime,
+          mode: entry.mode,
+          originalPath
+        }
+
+        const symlinkBuilder = options.symlinkBuilder ?? defaultSymlinkBuilder
+
+        yield async () => symlinkBuilder(symlink, blockstore, options)
       } else if (entry.path != null) {
         const dir: Directory = {
           path: entry.path,
@@ -135,3 +154,7 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder {
 function isFileCandidate (entry: any): entry is FileCandidate {
   return entry.content != null
 }
+
+function isSymLinkCandidate (entry: any): entry is SymlinkCandidate {
+  return entry.link != null
+}

packages/ipfs-unixfs-importer/src/dag-builder/symlink.ts

@@ -0,0 +1,37 @@
+import { encode, prepare } from '@ipld/dag-pb'
+import { UnixFS } from 'ipfs-unixfs'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { persist } from '../utils/persist.ts'
+import type { InProgressImportResult, Symlink, WritableStorage } from '../index.ts'
+import type { Version } from 'multiformats/cid'
+
+export interface SymlinkBuilderOptions {
+  cidVersion: Version
+  signal?: AbortSignal
+}
+
+export interface SymlinkBuilder {
+  (symlink: Symlink, blockstore: WritableStorage, options: SymlinkBuilderOptions): Promise<InProgressImportResult>
+}
+
+export const defaultSymlinkBuilder: SymlinkBuilder = async (symlink: Symlink, blockstore: WritableStorage, options: SymlinkBuilderOptions): Promise<InProgressImportResult> => {
+  const unixfs = new UnixFS({
+    type: 'symlink',
+    data: uint8ArrayFromString(symlink.link),
+    mtime: symlink.mtime,
+    mode: symlink.mode
+  })
+
+  const block = encode(prepare({ Data: unixfs.marshal() }))
+  const cid = await persist(block, blockstore, options)
+  const path = symlink.path
+
+  return {
+    cid,
+    path,
+    unixfs,
+    size: BigInt(block.length),
+    originalPath: symlink.originalPath,
+    block
+  }
+}

packages/ipfs-unixfs-importer/src/index.ts

@@ -105,7 +105,14 @@ export interface DirectoryCandidate {
   mode?: number
 }
 
-export type ImportCandidate = FileCandidate | DirectoryCandidate
+export interface SymlinkCandidate {
+  path: string
+  link: string
+  mtime?: Mtime
+  mode?: number
+}
+
+export type ImportCandidate = FileCandidate | DirectoryCandidate | SymlinkCandidate
 
 export interface File {
   content: AsyncIterable<Uint8Array>
@@ -122,6 +129,15 @@ export interface Directory {
   originalPath?: string
 }
 
+export interface Symlink {
+  link: string
+  path: string
+  name?: string
+  mtime?: Mtime
+  mode?: number
+  originalPath?: string
+}
+
 export interface ImportResult {
   cid: CID
   size: bigint
@@ -344,7 +360,7 @@ export interface ImporterOptions extends ProgressOptions<ImporterProgressEvents>
   fileBuilder?: FileBuilder
 }
 
-export type ImportCandidateStream = AsyncIterable<FileCandidate | DirectoryCandidate> | Iterable<FileCandidate | DirectoryCandidate>
+export type ImportCandidateStream = AsyncIterable<FileCandidate | DirectoryCandidate | SymlinkCandidate> | Iterable<FileCandidate | DirectoryCandidate | SymlinkCandidate>
 
 /**
  * The importer creates UnixFS DAGs and stores the blocks that make
@@ -374,7 +390,7 @@ export type ImportCandidateStream = AsyncIterable<FileCandidate | DirectoryCandi
  * ```
  */
 export async function * importer (source: ImportCandidateStream, blockstore: WritableStorage, options: ImporterOptions = {}): AsyncGenerator<ImportResult, void, unknown> {
-  let candidates: AsyncIterable<FileCandidate | DirectoryCandidate> | Iterable<FileCandidate | DirectoryCandidate>
+  let candidates: ImportCandidateStream
 
   if (Symbol.asyncIterator in source || Symbol.iterator in source) {
     candidates = source

packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.ts

@@ -8,6 +8,7 @@ import { balanced, flat, trickle } from '../src/layout/index.ts'
 import randomByteStream from './helpers/finite-pseudorandom-byte-stream.ts'
 import type { ImporterOptions } from '../src/index.ts'
 import type { FileLayout } from '../src/layout/index.ts'
+import type { Blockstore } from 'interface-blockstore'
 
 const strategies: Record<'flat' | 'trickle' | 'balanced', FileLayout> = {
   flat: flat(),
@@ -101,3 +102,37 @@ describe('go-ipfs auto-sharding interop', function () {
     expect(result.cid.toString()).to.be.equal('bafybeigyvxs6og5jbmpaa43qbhhd5swklqcfzqdrtjgfh53qjon6hpjaye')
   })
 })
+
+describe('go-ipfs interop', () => {
+  let block: Blockstore
+
+  beforeEach(() => {
+    block = new MemoryBlockstore()
+  })
+
+  it('should store the same metadata for symlinks', async () => {
+    const expected = {
+      'bar/hello.txt': 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o',
+      bar: 'QmfLiVjH2vujCVP2e75zyzBYmpcjktmDeU1YBz6Ct8BBsc',
+      'hello.txt': 'QmPFxdC7JLKj3Q6L6CFfVH9S9A26csjMwPx7Cmc7DFVkHC',
+      '': 'QmRE9CD9L1LV5i2AER98EfvigN31wcRZNAU99mpkdHqj6p'
+    }
+
+    const received: Record<string, string> = {}
+
+    for await (const result of importer([{
+      path: 'bar/hello.txt',
+      content: uint8ArrayFromString('hello world\n')
+    }, {
+      path: 'hello.txt',
+      link: 'bar/hello.txt'
+    }], block, {
+      profile: 'unixfs-v0-2015',
+      wrapWithDirectory: true
+    })) {
+      received[`${result.path}`] = result.cid.toString()
+    }
+
+    expect(received).to.deep.equal(expected)
+  })
+})

packages/ipfs-unixfs/src/index.ts

@@ -314,7 +314,7 @@ class UnixFS {
     return PBData.encode({
       Type: type,
       Data: data,
-      filesize: this.isDirectory() ? undefined : this.fileSize(),
+      filesize: this.type === 'file' || this.type === 'raw' ? this.fileSize() : undefined,
       blocksizes: this.blockSizes,
       hashType: this.hashType,
       fanout: this.fanout,