proof-of-concept: mm support

src/Companion.cpp

@@ -133,6 +133,9 @@
 #include "factories/oot/OoTPathFactory.h"
 #include "factories/oot/OoTCutsceneFactory.h"
 #include "factories/oot/OoTAudioFactory.h"
+#include "factories/oot/MMTextureAnimationFactory.h"
+#include "factories/oot/MMTextFactory.h"
+#include "factories/oot/MMKeyFrameFactory.h"
 #endif
 
 #ifdef NAUDIO_SUPPORT
@@ -327,6 +330,33 @@ void Companion::Init(const ExportType type, std::atomic<size_t>& assetCount, boo
     this->RegisterFactory("OOT:CUTSCENE", std::make_shared<OoT::OoTCutsceneFactory>());
     this->RegisterFactory("OOT:PATH", std::make_shared<OoT::OoTPathFactory>());
     this->RegisterFactory("OOT:AUDIO", std::make_shared<OoT::OoTAudioFactory>());
+
+    // Majora's Mask runs on the same engine and shares most asset formats with
+    // Ocarina of Time, so MM: names resolve to the OoT factories for now. Which of
+    // these actually produce byte-identical output is being measured type by type;
+    // the ones that do are what moves to a shared zelda64/ namespace, and the ones
+    // that don't get real MM implementations.
+    //
+    // MM-only types with no OoT counterpart -- TEXTURE_ANIMATION, KEYFRAME_ANIMATION,
+    // KEYFRAME_SKELETON -- are deliberately absent: nothing here can serve them.
+    this->RegisterFactory("MM:ARRAY", std::make_shared<OoT::OoTArrayFactory>());
+    this->RegisterFactory("MM:MTX", std::make_shared<OoT::OoTMtxFactory>());
+    this->RegisterFactory("MM:SKELETON", std::make_shared<OoT::OoTSkeletonFactory>());
+    this->RegisterFactory("MM:LIMB", std::make_shared<OoT::OoTLimbFactory>());
+    this->RegisterFactory("MM:ANIMATION", std::make_shared<OoT::OoTAnimationFactory>());
+    this->RegisterFactory("MM:CURVE_ANIMATION", std::make_shared<OoT::OoTCurveAnimationFactory>());
+    this->RegisterFactory("MM:PLAYER_ANIMATION", std::make_shared<OoT::OoTPlayerAnimationHeaderFactory>());
+    this->RegisterFactory("MM:PLAYER_ANIMATION_DATA", std::make_shared<OoT::OoTPlayerAnimationDataFactory>());
+    this->RegisterFactory("MM:COLLISION", std::make_shared<OoT::OoTCollisionFactory>());
+    this->RegisterFactory("MM:TEXT", std::make_shared<OoT::MMTextFactory>());
+    this->RegisterFactory("MM:SCENE", std::make_shared<OoT::OoTSceneFactory>());
+    this->RegisterFactory("MM:ROOM", std::make_shared<OoT::OoTSceneFactory>());
+    this->RegisterFactory("MM:CUTSCENE", std::make_shared<OoT::OoTCutsceneFactory>());
+    this->RegisterFactory("MM:PATH", std::make_shared<OoT::OoTPathFactory>());
+    this->RegisterFactory("MM:TEXTURE_ANIMATION", std::make_shared<OoT::MMTextureAnimationFactory>());
+    this->RegisterFactory("MM:KEYFRAME_SKELETON", std::make_shared<OoT::MMKeyFrameSkelFactory>());
+    this->RegisterFactory("MM:KEYFRAME_ANIMATION", std::make_shared<OoT::MMKeyFrameAnimFactory>());
+    this->RegisterFactory("MM:AUDIO", std::make_shared<OoT::OoTAudioFactory>());
 #endif
 
 #ifdef BUILD_UI
@@ -528,6 +558,26 @@ void Companion::ParseModdingConfig() {
     }
 }
 
+// Resolve an explicit `compression:` override from a file's :config:. Formats with
+// no magic of their own (CmpDma containers) cannot be sniffed by
+// GetCompressionType, so a file has to name them. Returns nullopt when the key is
+// absent, leaving auto-detection in charge.
+static std::optional<CompressionType> ExplicitCompressionType(const YAML::Node& config) {
+    if (!config || !config["compression"]) {
+        return std::nullopt;
+    }
+    const auto name = config["compression"].as<std::string>();
+    if (name == "CMPDMA") {
+        return CompressionType::CMPDMA;
+    }
+    if (name == "NONE") {
+        return CompressionType::None;
+    }
+    throw std::runtime_error("Unknown `compression:` value \"" + name +
+                             "\".\n\nSupported: CMPDMA, NONE. Omit the key to auto-detect from the "
+                             "file's magic (MIO0/Yay0/Yay1/Yaz0).");
+}
+
 void Companion::ParseCurrentFileConfig(YAML::Node node, std::atomic<size_t>& assetCount) {
     if (node["external_files"]) {
         auto externalFiles = node["external_files"];
@@ -620,16 +670,20 @@ void Companion::ParseCurrentFileConfig(YAML::Node node, std::atomic<size_t>& ass
         // Set global variables for segmented data
         if (segments.IsSequence() && segments.size()) {
             if (segments[0].IsSequence() && segments[0].size() == 2) {
-                gCurrentSegmentNumber = segments[0][0].as<uint32_t>();
-                gCurrentFileOffset = segments[0][1].as<uint32_t>();
+                SetSegmentInfo(segments);
+
                 gCurrentCompressionType = Decompressor::GetCompressionType(this->gRomData, gCurrentFileOffset);
+                if (const auto explicitType = ExplicitCompressionType(node)) {
+                    gCurrentCompressionType = *explicitType;
+                }
                 if (node["no_compression"]) {
                     gCurrentCompressionType = CompressionType::None;
                 }
             } else {
                 throw std::runtime_error(
-                    "Incorrect yaml syntax for segments.\n\nThe yaml expects:\n:config:\n  segments:\n  - [<segment>, "
-                    "<file_offset>]\n\nLike so:\nsegments:\n  - [0x06, 0x821D10]");
+                "Incorrect yaml syntax for segments.\n\nThe yaml expects:\n:config:\n  segments:\n  - [<segment>, "
+                "<file_offset>] or - [<segment>, "
+                "<file_name>] \n\nLike so:\nsegments:\n  - [0x06, 0x821D10] or [0x06, object_jya_obj");
             }
         }
 
@@ -638,13 +692,14 @@ void Companion::ParseCurrentFileConfig(YAML::Node node, std::atomic<size_t>& ass
             auto segment = segments[i];
             if (segment.IsSequence() && segment.size() == 2) {
                 const auto id = segment[0].as<uint32_t>();
-                const auto replacement = segment[1].as<uint32_t>();
+                const auto replacement = GetFileOffsetFromNodeStr(segment[1].as<std::string>());
                 this->gConfig.segment.local[id] = replacement;
                 SPDLOG_DEBUG("Segment {} replaced with 0x{:X}", id, replacement);
             } else {
                 throw std::runtime_error(
                     "Incorrect yaml syntax for segments.\n\nThe yaml expects:\n:config:\n  segments:\n  - [<segment>, "
-                    "<file_offset>]\n\nLike so:\nsegments:\n  - [0x06, 0x821D10]");
+                    "<file_offset>] or - [<segment>, "
+                    "<file_name>] \n\nLike so:\nsegments:\n  - [0x06, 0x821D10] or [0x06, object_jya_obj");
             }
         }
     }
@@ -1183,6 +1238,22 @@ void Companion::ProcessExportFile() {
     }
 }
 
+void Companion::SetSegmentInfo(const YAML::Node& segments) {
+    gCurrentSegmentNumber = segments[0][0].as<uint32_t>();
+
+    const auto offsetNode = segments[0][1].as<std::string>();
+    gCurrentFileOffset = GetFileOffsetFromNodeStr(offsetNode);
+}
+
+uint32_t Companion::GetFileOffsetFromNodeStr(const std::string& str) const {
+    // IsValidHex demands an 0x prefix and at least three characters, so a bare
+    // "0" would fall through to the filelist and throw. IsValidOffset covers it.
+    if (StringHelper::IsValidOffset(str)) {
+        return strtoul(str.c_str(), nullptr, 16);
+    }
+    return GetFileOffsetFromName(str);
+}
+
 void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
     // Reset per-file state so segment/offset settings from a previous file don't
     // bleed into this file's Phase 1 gAddrMap registration.
@@ -1201,16 +1272,20 @@ void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
     if (auto segments = root[":config"]["segments"]) {
         if (segments.IsSequence() && segments.size() > 0) {
             if (segments[0].IsSequence() && segments[0].size() == 2) {
-                gCurrentSegmentNumber = segments[0][0].as<uint32_t>();
-                gCurrentFileOffset = segments[0][1].as<uint32_t>();
+                SetSegmentInfo(segments);
+
                 gCurrentCompressionType = Decompressor::GetCompressionType(this->gRomData, gCurrentFileOffset);
+                if (const auto explicitType = ExplicitCompressionType(root[":config"])) {
+                    gCurrentCompressionType = *explicitType;
+                }
                 if (root[":config"]["no_compression"]) {
                     gCurrentCompressionType = CompressionType::None;
                 }
             } else {
                 throw std::runtime_error(
                     "Incorrect yaml syntax for segments.\n\nThe yaml expects:\n:config:\n  segments:\n  - [<segment>, "
-                    "<file_offset>]\n\nLike so:\nsegments:\n  - [0x06, 0x821D10]");
+                    "<file_offset>] or - [<segment>, "
+                    "<file_name>] \n\nLike so:\nsegments:\n  - [0x06, 0x821D10] or [0x06, object_jya_obj");
             }
         }
     }
@@ -1234,17 +1309,29 @@ void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
             continue;
         }
 
+        auto offset = GetFileOffsetFromNodeStr(node["offset"].as<std::string>());
+
         if (gCurrentSegmentNumber) {
-            if (IS_SEGMENTED(node["offset"].as<uint32_t>()) == false) {
-                node["offset"] = (gCurrentSegmentNumber << 24) | node["offset"].as<uint32_t>();
+
+            if (IS_SEGMENTED(offset) == false) {
+                offset = (gCurrentSegmentNumber << 24) | offset;
+                node["offset"] = offset;
+
             }
         }
 
         if (!gCurrentVirtualPath.empty()) {
             node["path"] = gCurrentVirtualPath;
         }
 
-        this->gAddrMap[this->gCurrentFile][node["offset"].as<uint32_t>()] = std::make_tuple(output, node);
+        // A `duplicate_of` node is a second copy of an asset already declared at
+        // this offset under another name, so it must not claim the address --
+        // pointers to that offset belong to the original.
+        if (node["duplicate_of"]) {
+            continue;
+        }
+
+        this->gAddrMap[this->gCurrentFile][offset] = std::make_tuple(output, node);
     }
 
     // Stupid hack because the iteration broke the assets
@@ -1414,9 +1501,32 @@ void Companion::Process(std::atomic<size_t>& assetCount) {
         }
     }
     this->gAssetPath = (this->gSourceDirectory / rom["path"].as<std::string>()).string();
+
+    if (rom["filelist"]) {
+        const std::string filelistPath = (this->gSourceDirectory / rom["filelist"].as<std::string>()).string();
+        if (!fs::exists(filelistPath)) {
+            SPDLOG_ERROR("A filelist was specified but the file doesn't exist");
+            return;
+        }
+        ParseFilelist(filelistPath);
+    }
+
     auto opath = cfg["output"];
     auto gbi = cfg["gbi"];
     auto gbi_floats = cfg["gbi_floats"];
+
+    // OoT and MM share this engine and most of their asset formats; the places
+    // they diverge need to know which one they are looking at.
+    this->gConfig.zelda64Game = Zelda64Game::OoT;
+    if (cfg["game"]) {
+        const auto game = cfg["game"].as<std::string>();
+        if (game == "MM") {
+            this->gConfig.zelda64Game = Zelda64Game::MM;
+        } else if (game != "OOT") {
+            throw std::runtime_error("Unknown `game:` value \"" + game +
+                                     "\".\n\nSupported: OOT (default), MM.");
+        }
+    }
     auto modding_path = opath && opath["modding"] ? opath["modding"].as<std::string>() : "modding";
 
     if (!this->gDestinationDirectory.empty() && !fs::exists(this->gDestinationDirectory)) {
@@ -2590,3 +2700,16 @@ bool Companion::GetCompressedSegmentOffset(uint32_t* addr) {
     }
     return false;
 }
+
+void Companion::ParseFilelist(const std::string& filelistPath) {
+    YAML::Node root = YAML::LoadFile(filelistPath);
+
+    for (const auto f : root["Files"]) {
+        for (const auto& kv : f) {
+            const auto file = kv.first.as<std::string>();
+            const auto offset = kv.second.as<uint32_t>();
+            gFileOffsets[file] = offset;
+        }
+
+    }
+}

src/Companion.h

@@ -101,8 +101,18 @@ struct GBIConfig {
     bool useFloats = false;
 };
 
+// Which Zelda 64 title a rom is. Ocarina of Time and Majora's Mask run on the same
+// engine and share most asset formats, but a few structures carry extra fields in
+// MM -- pathways, for one -- so shared code has to be able to tell them apart.
+// Declared per rom with `game: MM` in its config; defaults to OoT.
+enum class Zelda64Game {
+    OoT,
+    MM,
+};
+
 struct TorchConfig {
     GBIConfig gbi;
+    Zelda64Game zelda64Game = Zelda64Game::OoT;
     SegmentConfig segment;
     std::string outputPath;
     std::string moddingPath;
@@ -189,6 +199,8 @@ class Companion {
     std::string GetDestRelativeOutputPath() { return RelativePathToDestDir(GetOutputPath()); }
 
     GBIVersion GetGBIVersion() const { return this->gConfig.gbi.version; }
+    Zelda64Game GetZelda64Game() const { return this->gConfig.zelda64Game; }
+    bool IsMajorasMask() const { return this->gConfig.zelda64Game == Zelda64Game::MM; }
     GBIMinorVersion GetGBIMinorVersion() const { return  this->gConfig.gbi.subversion; }
     std::unordered_map<std::string, std::vector<YAML::Node>> GetCourseMetadata() { return this->gCourseMetadata; }
     std::optional<std::string> GetEnumFromValue(const std::string& key, int id);
@@ -217,8 +229,9 @@ class Companion {
     const std::vector<std::tuple<std::string, YAML::Node>>* GetNodesByTypeRef(const std::string& type, bool includeAutogen = false);
     std::string GetSymbolFromAddr(uint32_t addr, bool validZero = false);
 
-    std::optional<std::uint32_t> GetFileOffset(void) const { return this->gCurrentFileOffset; };
-    std::optional<std::uint32_t> GetCurrSegmentNumber(void) const { return this->gCurrentSegmentNumber; };
+    std::string GetCurrentFile(void) { return this->gCurrentFile; }
+    std::optional<std::uint32_t> GetFileOffsetFromName(void) const { return this->gCurrentFileOffset; };
+    std::uint32_t GetCurrSegmentNumber(void) const { return this->gCurrentSegmentNumber; };
     CompressionType GetCurrCompressionType(void) const { return this->gCurrentCompressionType; };
     std::optional<std::uint32_t> GetCurrentCompressedSize(void) const { return this->gCurrentCompressedSize; };
     std::optional<VRAMEntry> GetCurrentVRAM(void) const { return this->gCurrentVram; };
@@ -253,6 +266,7 @@ class Companion {
     bool GetCompressedSegmentOffset(uint32_t* addr);
 
     void SetSingleYMLPath(const std::string& path) { this->gSingleYMLPath = path; }
+    uint32_t GetFileOffsetFromName(const std::string& file) const { return this->gFileOffsets.at(file); }
 
 #ifdef BUILD_UI
     void RegisterUIFactory(const std::string& type, const std::shared_ptr<BaseFactoryUI>& factory);
@@ -301,6 +315,7 @@ class Companion {
     std::vector<std::string> gCurrentExternalFiles;
     std::unordered_map<int, std::string> gManualSegments;
     std::unordered_set<std::string> gProcessedFiles;
+    std::unordered_map<std::string, uint32_t> gFileOffsets;
 
     std::unordered_map<std::string, std::vector<char>> gCompanionFiles;
     std::vector<std::pair<std::string, std::vector<char>>> gArchiveFiles;
@@ -341,4 +356,7 @@ class Companion {
     void LoadYAMLRecursively(const std::string &dirPath, std::vector<YAML::Node> &result, bool skipRoot);
     std::vector<fs::directory_entry> GetAssetYMLs(YAML::Node& rom) const;
     std::optional<ParseResultData> ParseNode(YAML::Node& node, std::string& name);
+    void ParseFilelist(const std::string& filelistPath);
+    void SetSegmentInfo(const YAML::Node& segments);
+    uint32_t GetFileOffsetFromNodeStr(const std::string& str) const;
 };

src/factories/ResourceType.h

@@ -82,6 +82,12 @@ enum class ResourceType {
     OoTBackground = 0x4F424749,      // OBGI
     OoTSceneCommand = 0x4F52434D,    // ORCM
 
+    // MM
+    MMTextureAnimation = 0x4F54414E, // OTAN
+    MMText = 0x4F54584D,             // OTXM
+    MMKeyFrameAnim = 0x4F4B4641,     // OKFA
+    MMKeyFrameSkel = 0x4F4B4653,     // OKFS
+
     // BK64
     BKSprite = 0x424B5350,         // BKSP
     BKAnimation = 0x424B414E,      // BKAN