feat: Normalize unicode passwords

build.zig

@@ -4,10 +4,18 @@ pub fn build(b: *std.Build) void {
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
 
+    // Add dependencies
+    const clap = b.dependency("clap", .{});
+    const zg = b.dependency("zg", .{});
+
     // root module
     const mod = b.addModule("z-v4-converter", .{
         .root_source_file = b.path("src/root.zig"),
         .target = target,
+        .imports = &.{
+            .{ .name = "Normalize", .module = zg.module("Normalize") },
+            .{ .name = "code_point", .module = zg.module("code_point") },
+        },
     });
 
     // build zon module - to obtain the --version for cli
@@ -24,52 +32,26 @@ pub fn build(b: *std.Build) void {
             .root_source_file = b.path("src/main.zig"),
             .target = target,
             .optimize = optimize,
-            // List of modules available for import in source files part of the
-            // root module.
             .imports = &.{
                 .{ .name = "z-v4-converter", .module = mod },
                 .{ .name = "build.zig.zon", .module = build_zig_zon },
+                .{ .name = "zig-clap", .module = clap.module("clap") },
             },
         }),
     });
 
-    // Add zig-clap dependency
-    const clap = b.dependency("clap", .{});
-    exe.root_module.addImport("zig-clap", clap.module("clap"));
-
-    // This declares intent for the executable to be installed into the
-    // install prefix when running `zig build` (i.e. when executing the default
-    // step). By default the install prefix is `zig-out/` but can be overridden
-    // by passing `--prefix` or `-p`.
     b.installArtifact(exe);
 
-    // This creates a top level step. Top level steps have a name and can be
-    // invoked by name when running `zig build` (e.g. `zig build run`).
-    // This will evaluate the `run` step rather than the default step.
-    // For a top level step to actually do something, it must depend on other
-    // steps (e.g. a Run step, as we will see in a moment).
+    // Run step for main executeable
     const run_step = b.step("run", "Run the app");
-
-    // This creates a RunArtifact step in the build graph. A RunArtifact step
-    // invokes an executable compiled by Zig. Steps will only be executed by the
-    // runner if invoked directly by the user (in the case of top level steps)
-    // or if another step depends on it, so it's up to you to define when and
-    // how this Run step will be executed. In our case we want to run it when
-    // the user runs `zig build run`, so we create a dependency link.
     const run_cmd = b.addRunArtifact(exe);
     run_step.dependOn(&run_cmd.step);
-
-    // By making the run step depend on the default step, it will be run from the
-    // installation directory rather than directly from within the cache directory.
     run_cmd.step.dependOn(b.getInstallStep());
-
     // This allows the user to pass arguments to the application in the build
     // command itself, like this: `zig build run -- arg1 arg2 etc`
-    if (b.args) |args| {
-        run_cmd.addArgs(args);
-    }
+    if (b.args) |args| run_cmd.addArgs(args);
 
-    // add Tests
+    // Test module for tests/all.zig
     const mod_tests = b.addTest(.{
         .root_module = b.createModule(.{
             .root_source_file = b.path("tests/all.zig"),
@@ -80,35 +62,25 @@ pub fn build(b: *std.Build) void {
         }),
     });
 
-    // A run step that will run the test executable.
     const run_mod_tests = b.addRunArtifact(mod_tests);
 
-    // Creates an executable that will run `test` blocks from the executable's
-    // root module. Note that test executables only test one module at a time,
-    // hence why we have to create two separate ones.
-    const exe_tests = b.addTest(.{
-        .root_module = exe.root_module,
+    // Test module for src/root.zig (to pickup all tests in src/)
+    const root_tests = b.addTest(.{
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("src/root.zig"),
+            .target = target,
+            .imports = &.{
+                .{ .name = "z-v4-converter", .module = mod },
+                .{ .name = "Normalize", .module = zg.module("Normalize") },
+                .{ .name = "code_point", .module = zg.module("code_point") },
+            },
+        }),
     });
 
-    // A run step that will run the second test executable.
-    const run_exe_tests = b.addRunArtifact(exe_tests);
+    const run_root_tests = b.addRunArtifact(root_tests);
 
-    // A top level step for running all tests. dependOn can be called multiple
-    // times and since the two run steps do not depend on one another, this will
-    // make the two of them run in parallel.
+    // Top-level test step that runs all tests
     const test_step = b.step("test", "Run tests");
     test_step.dependOn(&run_mod_tests.step);
-    test_step.dependOn(&run_exe_tests.step);
-
-    // Just like flags, top level steps are also listed in the `--help` menu.
-    //
-    // The Zig build system is entirely implemented in userland, which means
-    // that it cannot hook into private compiler APIs. All compilation work
-    // orchestrated by the build system will result in other Zig compiler
-    // subcommands being invoked with the right flags defined. You can observe
-    // these invocations when one fails (or you pass a flag to increase
-    // verbosity) to validate assumptions and diagnose problems.
-    //
-    // Lastly, the Zig build system is relatively simple and self-contained,
-    // and reading its source code will allow you to master it.
+    test_step.dependOn(&run_root_tests.step);
 }

build.zig.zon

@@ -13,6 +13,11 @@
             .url = "git+https://github.com/Hejsil/zig-clap#b7e3348ed60f99ba32c75aa707ff7c87adc31b36",
             .hash = "clap-0.11.0-oBajB-TnAQC7yPLnZRT5WzHZ_4Ly4dX2OILskli74b9H",
         },
+        // TODO: Update zg dependency to upstream https://codeberg.org/atman/zg
+        .zg = .{
+            .url = "git+https://codeberg.org/chaten/zg.git#749197a3f9d25e211615960c02380a3d659b20f9",
+            .hash = "zg-0.15.1-oGqU3M0-tALZCy7boQS86znlBloyKx6--JriGlY0Paa9",
+        },
     },
     .paths = .{ "build.zig", "build.zig.zon", "src", "tests", "LICENSE-APACHE-2.0", "LICENSE-MIT", "README.md" },
 }

src/conversion.zig

@@ -1,6 +1,9 @@
 const std = @import("std");
 const types = @import("types.zig");
 const parsing = @import("parsing.zig");
+const Normalize = @import("Normalize");
+const code_point = @import("code_point");
+const testing = std.testing;
 
 const ConversionOptions = types.ConversionOptions;
 const ConversionContext = types.ConversionContext;
@@ -60,12 +63,73 @@ fn readKeystoreAndPassword(allocator: std.mem.Allocator, pk: []const u8, ctx: *C
 
     const password_file = try ctx.password_dir.openFile(password_path, .{});
     defer password_file.close();
-    const password_content = try password_file.readToEndAlloc(allocator, try password_file.getEndPos());
-    errdefer allocator.free(password_content);
+
+    const rawPassword = try password_file.readToEndAlloc(allocator, try password_file.getEndPos());
+    defer allocator.free(rawPassword);
+
+    const normalize_password = try normalizePassword(allocator, rawPassword);
+    errdefer allocator.free(normalize_password);
 
     return KeystoreData{
         .keystore_content = keystore_content,
-        .password_content = password_content,
+        .password_content = normalize_password,
         .allocator = allocator,
     };
 }
+
+fn normalizePassword(allocator: std.mem.Allocator, password: []const u8) ![]u8 {
+    if (password.len == 0) return allocator.dupe(u8, "");
+
+    // Step 1: NFKD normalize
+    var normalize = try Normalize.init(allocator);
+    defer normalize.deinit(allocator);
+
+    var nfkd_result = try normalize.nfkd(allocator, password);
+    defer nfkd_result.deinit(allocator);
+
+    // Step 2: Strip disallowed control codes
+    var buf: std.ArrayList(u8) = .empty;
+    defer buf.deinit(allocator);
+
+    var it: code_point.Iterator = .init(nfkd_result.slice);
+    while (it.next()) |cp| {
+        if (isAllowedCodepoint(cp.code)) {
+            // Encode codepoint back into UTF-8
+            var tmp: [4]u8 = undefined;
+            const len = std.unicode.utf8Encode(cp.code, &tmp) catch continue;
+            try buf.appendSlice(allocator, tmp[0..len]);
+        }
+    }
+
+    // Step 3: Return owned slice (UTF-8 encoded)
+    return buf.toOwnedSlice(allocator);
+}
+
+/// Only allow codepoints that are NOT C0, C1, or Delete (0x00–0x1F, 0x7F, 0x80–0x9F)
+fn isAllowedCodepoint(cp: u21) bool {
+    return !((cp <= 0x1F) or (cp == 0x7F) or (cp >= 0x80 and cp <= 0x9F));
+}
+
+test "normalizePassword produces correct output" {
+    const allocator = std.testing.allocator;
+    const password = "𝔱𝔢𝔰𝔱𝔭𝔞𝔰𝔰𝔴𝔬𝔯𝔡🔑";
+
+    const normalized = try normalizePassword(allocator, password);
+    defer allocator.free(normalized);
+
+    // Convert to hex string for comparison
+    const hex_encoded = try std.fmt.allocPrint(allocator, "0x{x}", .{normalized});
+    defer allocator.free(hex_encoded);
+
+    try testing.expectEqualStrings(hex_encoded, "0x7465737470617373776f7264f09f9491");
+}
+
+test "normalizePassword with empty string produces correct output" {
+    const allocator = std.testing.allocator;
+    const password = "";
+
+    const normalized = try normalizePassword(allocator, password);
+    defer allocator.free(normalized);
+
+    try testing.expectEqualStrings(normalized, "");
+}

src/root.zig

@@ -15,3 +15,10 @@ pub const KeystoreData = types.KeystoreData;
 // Re-export main functions
 pub const validatePaths = validation.validatePaths;
 pub const convertKeystores = conversion.convertKeystores;
+
+test {
+    _ = @import("types.zig");
+    _ = @import("validation.zig");
+    _ = @import("conversion.zig");
+    _ = @import("parsing.zig");
+}