KM_TextManager.pas

unit KM_TextManager;
{$I KM_CompilerDirectives.inc}
interface
uses
  System.Classes, Vcl.Dialogs, System.Math, Vcl.Clipbrd, System.Generics.Defaults,
  System.StrUtils, System.SysUtils, System.Types,
  KM_ResLocales,
  KM_TextLines;


type
  TKMTargetGame = (tgUnknown, tgKaMRemake, tgKnightsProvince);

  TKMLibraryType = (
    ltGame,
    ltMissions
  );

  TKMClipboardExport = (
    ceSimple,     // Just columns with locales
    ceLastChanged // Include LastChanged for each locale
  );

  TKMTextManager = class
  private const
    COL_NAME_TAG = 'Tag';
    COL_NAME_NEEDS_UPDATE = 'Needs update';
    COL_NAME_DESC = 'Description';
  private class var
    LOCALE_DEFAULT: Integer;
  private
    fLocales: TKMResLocales;
    fWorkDir: string;

    fTextPath: string;
    fTagsPath: string; // We use consts only for ingame library, others don't need them
    fMetaPath: string; // We use meta only for ingame library, others don't need it

    fLines: TKMLines;
    fTargetGame: TKMTargetGame;
    fLibType: TKMLibraryType;

    // For copy/paste
    fBuffer: array of string;

    fHasChanges: Boolean;

    function GetCount: Integer;
    function GetItem(aIndex: Integer): TKMLine;
    procedure LoadMeta(const aFilename: string);
    procedure SaveMeta(const aFilename: string);
    procedure TagsAutoName(const aPath: string);
    function ToClipboardHeader(aLocales: TByteSet; aExport: TKMClipboardExport): string;
    function ToClipboardBody(aLocales: TByteSet; aExport: TKMClipboardExport): string;
    procedure ClipboardColumnsToLocIndex(aLine: string; out aLocIndex: TArray<Integer>);
  public const
    // Path matches for KMR and KP
    GAME_TEXT_PATH = 'data\text\text.%s.libx';

    constructor Create(aTargetGame: TKMTargetGame; aLocales: TKMResLocales; const aWorkDir, aTagsPath, aMetaPath: string);
    destructor Destroy; override;

    procedure Load4(const aTextPath: string; aLocales: TByteSet);
    procedure Save(aSortById: Boolean);

    property Count: Integer read GetCount;
    property Items[aIndex: Integer]: TKMLine read GetItem; default;
    property HasChanges: Boolean read fHasChanges write fHasChanges;

    procedure TextCopy(aIndex: Integer);
    procedure TextPaste(aIndex: Integer);
    procedure EraseAllStringsButEng(aIndex: Integer);

    procedure Delete(aIndex: Integer);
    procedure Insert(aIndex: Integer);
    procedure InsertSeparator(aIndex: Integer);
    procedure MoveUp(aIndex: Integer);
    procedure MoveDown(aIndex: Integer);
    procedure SortByIndex;
    procedure SortByTag;
    procedure CompactIndexes;

    procedure ToClipboard(aLocales: TByteSet; aExport: TKMClipboardExport);
    procedure ToClipboardAll(aList: TStringList; aLocales: TByteSet);
    procedure FromClipboard;
    procedure FromClipboardAll(aSortById: Boolean);

    procedure ListMismatchingAll(aFolders: TStringList; aList: TStringList; aLocales: TByteSet);
  end;


implementation
uses
  KromStringUtils,
  KM_IoXml;


{ TKMTextManager }
constructor TKMTextManager.Create(aTargetGame: TKMTargetGame; aLocales: TKMResLocales; const aWorkDir, aTagsPath, aMetaPath: string);
begin
  inherited Create;

  fLines := TKMLines.Create;

  fTargetGame := aTargetGame;
  fLocales := aLocales;
  fWorkDir := aWorkDir;
  fTagsPath := fWorkDir + aTagsPath;
  fMetaPath := fWorkDir + aMetaPath;

  TKMLine.LOCALE_COUNT := fLocales.Count;
  LOCALE_DEFAULT := fLocales.IndexByCode(TKMResLocales.DEFAULT_LOCALE);
  TKMLine.LOCALE_DEFAULT := LOCALE_DEFAULT;
end;


destructor TKMTextManager.Destroy;
begin
  FreeAndNil(fLines);

  inherited;
end;


procedure TKMTextManager.Load4(const aTextPath: string; aLocales: TByteSet);
var
  I: Integer;
begin
  fTextPath := fWorkDir + aTextPath;

  fLines.Clear;

  // See if this is a game libx or a mission
  if Pos(GAME_TEXT_PATH, fTextPath) <> 0 then
    fLibType := ltGame
  else
    fLibType := ltMissions;

  // If we have consts - good, use them
  if fLibType = ltGame then
    fLines.LoadGameConsts(fTagsPath);

  for I := 0 to fLocales.Count - 1 do
    if (fTargetGame = tgKaMRemake)
    or ((fTargetGame = tgKnightsProvince) and (fLibType = ltGame)) then
      fLines.LoadLibx(Format(fTextPath, [fLocales[I].Code]), I)
    else
      fLines.LoadLibxKP(Format(fTextPath, [fLocales[I].Code]), I);

  if fLibType = ltGame then
    LoadMeta(fMetaPath);

  if (fTargetGame = tgKaMRemake) and (fLibType = ltMissions) then
    TagsAutoName(aTextPath); // Name tags just for UI, they wont be saved

  fHasChanges := False;
end;


procedure TKMTextManager.Save(aSortById: Boolean);
var
  I: Integer;
  fname: string;
begin
  // Save Libx files
  for I := 0 to fLocales.Count - 1 do
  begin
    fname := Format(fTextPath, [fLocales[I].Code]);

    if (fTargetGame = tgKaMRemake)
    or ((fTargetGame = tgKnightsProvince) and (fLibType = ltGame)) then
      fLines.SaveLibx(fname, I, (fLibType = ltMissions), aSortById)
    else
      fLines.SaveLibxKP(fname, I, (fLibType = ltGame));
  end;

  // For the game also save the consts and meta
  if fLibType = ltGame then
  begin
    fLines.SaveGameConsts(fTagsPath);
    SaveMeta(fMetaPath);
  end;

  fHasChanges := False;
end;


procedure TKMTextManager.LoadMeta(const aFilename: string);
var
  I, K, L: Integer;
  xml: TKMXMLDocument;
  nTags, nLine, nLastChanged, nDesc, nLastEng: TXmlNode;
  tagName: string;
  lastEng: string;
  tagExists: Boolean;
begin
  if not FileExists(aFilename) then Exit;

  xml := TKMXMLDocument.Create;
  try
    xml.LoadFromFile(aFilename);

    nTags := xml.Root.Find('Tags');

    for I := 0 to nTags.ChildNodes.Count - 1 do
    begin
      nLine := nTags.ChildNodes[I];
      tagName := nLine.Name;
      nLastChanged := nLine.Find('LastChanged');

      L := fLines.IndexOfTag(tagName);

      tagExists := L <> -1;
      if tagExists then
      begin
        if nLastChanged <> nil then
          for K := 0 to fLocales.Count - 1 do
            if nLastChanged.HasAttribute(fLocales[K].Code) then
              fLines[L].SetLastChanged(K, nLastChanged.Attributes[fLocales[K].Code].AsString);

        nDesc := nLine.Find('Description');
        fLines[L].Description := nDesc.Attributes['Value'].AsString;

        nLastEng := nLine.Find('LastEng');
        lastEng := nLastEng.Attributes['Value'].AsString;
        if lastEng <> fLines[L].Strings[LOCALE_DEFAULT] then
          fLines[L].LastChanged[LOCALE_DEFAULT] := Now;
      end;
    end;
  finally
    xml.Free;
  end;
end;


procedure TKMTextManager.SaveMeta(const aFilename: string);
var
  I, K: Integer;
  xml: TKMXMLDocument;
  nTags, nLine, nLastChanged, nDesc, nLastEng: TXmlNode;
begin
  xml := TKMXMLDocument.Create;
  try
    nTags := xml.Root.AddChild('Tags');

    for I := 0 to fLines.Count - 1 do
    if not fLines[I].IsSpacer then
    begin
      nLine := nTags.AddChild(fLines[I].Tag);

      nLastChanged := nLine.AddChild('LastChanged');
      for K := 0 to fLocales.Count - 1 do
        nLastChanged.Attributes[fLocales[K].Code] := fLines[I].GetLastChanged(K);

      nDesc := nLine.AddChild('Description');
      nDesc.Attributes['Value'] := fLines[I].Description;

      nLastEng := nLine.AddChild('LastEng');
      nLastEng.Attributes['Value'] := fLines[I].Strings[LOCALE_DEFAULT];
    end;

    xml.SaveToFile(aFilename);
  finally
    xml.Free;
  end;
end;


procedure TKMTextManager.SortByIndex;
begin
  fLines.Sort(TComparer<TKMLine>.Construct(
    function (const aLeft, aRight: TKMLine): Integer
    begin
      Result := CompareValue(aLeft.Id, aRight.Id);
    end));

  fHasChanges := True;
end;


procedure TKMTextManager.SortByTag;
begin
  fLines.Sort(TComparer<TKMLine>.Construct(
    function (const aLeft, aRight: TKMLine): Integer
    begin
      Result := CompareStr(aLeft.Tag, aRight.Tag);
    end));

  fHasChanges := True;
end;


// Our goal is to keep everything in place but update the Ids
procedure TKMTextManager.CompactIndexes;
const
  SPACING = 5; // Handy for adding new strings
var
  I, K: Integer;
begin
  if fLibType = ltMissions then Exit;

  K := 0;
  for I := 0 to fLines.Count - 1 do
  if not fLines[I].IsSpacer then
  begin
    fLines[I].Id := K;
    Inc(K);
  end else
    Inc(K, SPACING);

  fHasChanges := True;
end;


function TKMTextManager.GetCount: Integer;
begin
  Result := fLines.Count;
end;


function TKMTextManager.GetItem(aIndex: Integer): TKMLine;
begin
  Result := fLines[aIndex];
end;


procedure TKMTextManager.TagsAutoName(const aPath: string);
begin
  fLines.TagsAutoName(aPath);

  fHasChanges := True;
end;


procedure TKMTextManager.TextCopy(aIndex: Integer);
var
  I: Integer;
begin
  SetLength(fBuffer, fLocales.Count);
  for I := 0 to fLocales.Count - 1 do
    fBuffer[I] := fLines[aIndex].Strings[I];
end;


procedure TKMTextManager.TextPaste(aIndex: Integer);
var
  I: Integer;
begin
  Assert(Length(fBuffer) = fLocales.Count);
  for I := 0 to fLocales.Count - 1 do
    fLines[aIndex].Strings[I] := fBuffer[I];

  fHasChanges := True;
end;


procedure TKMTextManager.EraseAllStringsButEng(aIndex: Integer);
var
  I: Integer;
begin
  for I := 0 to fLocales.Count - 1 do
  if I <> LOCALE_DEFAULT then
    fLines[aIndex].Strings[I] := '';

  fHasChanges := True;
end;


procedure TKMTextManager.Insert(aIndex: Integer);
begin
  fLines.Insert(aIndex, TKMLine.Create(fLines.Last.Id + 1, 'TX_NEW' + IntToStr(fLines.Last.Id + 1)));
end;


procedure TKMTextManager.Delete(aIndex: Integer);
begin
  fLines.Delete(aIndex);

  fHasChanges := True;
end;


procedure TKMTextManager.InsertSeparator(aIndex: Integer);
begin
  fLines.Insert(aIndex, TKMLine.CreateSpacer);

  fHasChanges := True;
end;


procedure TKMTextManager.MoveUp(aIndex: Integer);
begin
  if aIndex > 0 then
    // Can't move the top item up
    fLines.Move(aIndex, aIndex - 1);

  fHasChanges := True;
end;


procedure TKMTextManager.MoveDown(aIndex: Integer);
begin
  if aIndex < fLines.Count - 1 then
    // Can't move the bottom item down
    fLines.Move(aIndex, aIndex + 1);

  fHasChanges := True;
end;


function TKMTextManager.ToClipboardHeader(aLocales: TByteSet; aExport: TKMClipboardExport): string;
var
  I: Integer;
begin
  // First line, list lang names
  Result := COL_NAME_TAG;

  for I := 0 to fLocales.Count - 1 do
  if (aLocales = []) or (I in aLocales) then
    Result := Result + #9 + fLocales[I].Code;

  if aExport = ceLastChanged then
    Result := Result + #9 + COL_NAME_NEEDS_UPDATE;

  Result := Result + #9 + COL_NAME_DESC;

  Result := Result + sLineBreak;
end;


function TKMTextManager.ToClipboardBody(aLocales: TByteSet; aExport: TKMClipboardExport): string;
var
  I, K: Integer;
  s: string;
begin
  Result := '';

  // Do the export
  for I := 0 to fLines.Count - 1 do
  begin
    if not fLines[I].IsSpacer then
    begin
      Result := Result + fLines[I].Tag;

      for K := 0 to fLocales.Count - 1 do
      if (aLocales = []) or (K in aLocales) then
      begin
        s := fLines[I].Strings[K];

        s := EscapeTextForGoogleSheets(s);

        Result := Result + #9 + s;

        if (fLibType = ltGame) and (aExport = ceLastChanged) and (K <> LOCALE_DEFAULT) then
        begin
          if (fLines[I].LastChanged[LOCALE_DEFAULT] = 0) then
            Result := Result + #9 + '' // New line (not saved in Meta)
          else
          if (fLines[I].LastChanged[K] = 0) then
            Result := Result + #9 + '' // New localization (not saved in Meta)
          else
          if (fLines[I].LastChanged[K] < fLines[I].LastChanged[LOCALE_DEFAULT]) then
            Result := Result + #9 + 'UPDATED'
          else
            Result := Result + #9 + '';
        end;
      end;

      Result := Result + #9 + fLines[I].Description;
    end
    else
      Result := Result + '-';

    Result := Result + sLineBreak;
  end;
end;


// Copy to clipboard to be pasted into Excel/Sheets
// Tag Text Text Text
procedure TKMTextManager.ToClipboard(aLocales: TByteSet; aExport: TKMClipboardExport);
begin
  Clipboard.AsText := ToClipboardHeader(aLocales, aExport) + ToClipboardBody(aLocales, aExport);
end;


// Copy to clipboard to be pasted into Excel/Sheets
// Tag Text Text Text
procedure TKMTextManager.ToClipboardAll(aList: TStringList; aLocales: TByteSet);
  function TranslationPercent(aLineFrom, aLineTo: Integer): string;
  var
    I, K: Integer;
    fromStr, toStr: string;
  begin
    Result := '-';
    K := 0;
    fromStr := IntToStr(aLineFrom);
    toStr := IntToStr(aLineTo);
    for I := 0 to fLocales.Count - 1 do
    if (aLocales = []) or (I in aLocales) then
    begin
      Result := Result + #9;

      // Add % and a hint for localizations that can reference Eng
      if I <> LOCALE_DEFAULT then
        Result := Result + '=ROUND(COUNTA('+Chr(Ord('B')+K)+fromStr+':'+Chr(Ord('B')+K)+toStr+')/COUNTA(B'+fromStr+':B'+toStr+')*100,2)' + '&"% (use <== when translation is not needed)"';

      Inc(K);
    end;

    Result := Result + #9 + 'Conditional formatting range is `C3:C2600` and formula is `=AND(ISBLANK(C3), NOT (ISBLANK(B3)))`';
  end;
var
  localesCount: Integer;
  I: Integer;
  s: string;
  lineCount: Integer;
  exportMode: TKMClipboardExport;
begin
  // How many locales are we exporting (except Eng)
  localesCount := 0;
  for I := 0 to fLocales.Count - 1 do
  if (I <> LOCALE_DEFAULT) and ((aLocales = []) or (I in aLocales)) then
    Inc(localesCount);

  if localesCount = 1 then
    exportMode := ceLastChanged
  else
    exportMode := ceSimple;

  s := '';
  lineCount := 1;
  for I := 0 to aList.Count - 1 do
  begin
    s := s + '> ' + aList[I] + sLineBreak;
    Inc(lineCount);

    Load4(aList[I], aLocales);

    s := s + ToClipboardBody(aLocales, exportMode);
    Inc(lineCount, fLines.Count);
  end;

  Clipboard.AsText := ToClipboardHeader(aLocales, exportMode) + TranslationPercent(3, lineCount+1) + sLineBreak + s;
end;


procedure TKMTextManager.ClipboardColumnsToLocIndex(aLine: string; out aLocIndex: TArray<Integer>);
var
  ss: TStringDynArray;
  I: Integer;
begin
  ss := SplitString(aLine, #9);

  Assert(Length(ss) > 1, 'Column names line needs to have at least 2 columns (Tag and one Locale)');

  // Always skip first column (Tag)
  SetLength(aLocIndex, Length(ss) - 1);

  for I := 1 to High(ss) do
  if (ss[I] <> '')
  and (ss[I] <> TKMTextManager.COL_NAME_NEEDS_UPDATE)
  and (ss[I] <> TKMTextManager.COL_NAME_DESC) then
    aLocIndex[I - 1] := fLocales.IndexByCode(ss[I])
  else
  begin
    SetLength(aLocIndex, I-1);
    Break;
  end;
end;


// Update translations
procedure TKMTextManager.FromClipboard;
var
  sl: TStringList;
  locIndex: TArray<Integer>;
  I, K: Integer;
  tag: string;
  txt: array of string;
  idLine, idLoc: Integer;
  changesCount: Integer;
  s: string;
begin
  sl := TStringList.Create;
  sl.Text := Clipboard.AsText;

  // First line, get lang names
  ClipboardColumnsToLocIndex(sl[0], locIndex);

  SetLength(txt, fLocales.Count);

  // Other lines, get the strings
  changesCount := 0;
  for I := 1 to sl.Count - 1 do
  if StringFromString(sl[I], #9, 0) <> '-' then
  begin
    // Source data
    tag := StringFromString(sl[I], #9, 0);
    for K := 0 to High(locIndex) do
    begin
      s := StringFromString(sl[I], #9, K+1);
      if StartsStr(#39'=', s) then
        s := LeftStr(s, Length(s) - 1);

      txt[K] := s;
    end;

    // Now we have all data to update

    idLine := fLines.IndexOfTag(tag);

    for K := 0 to High(locIndex) do
    begin
      idLoc := locIndex[K];
      if fLines[idLine].Strings[idLoc] <> txt[K] then
      begin
        fLines[idLine].Strings[idLoc] := txt[K];
        Inc(changesCount);
      end;
    end;
  end;

  ShowMessage(Format('%d lines updated', [changesCount]));

  sl.Free;

  fHasChanges := True;
end;


procedure TKMTextManager.FromClipboardAll(aSortById: Boolean);
var
  sl: TStringList;
  locIndex: TArray<Integer>;
  I, K: Integer;
  tag: string;
  txt: array of string;
  idLine, idLoc: Integer;
  changesCount: Integer;
  s: string;
  fname: string;
  filesCount: Integer;
  dt: TDateTime;
begin
  dt := Now; // Query once, so we dont get on a "second change" moment

  sl := TStringList.Create;
  sl.Text := Clipboard.AsText;

  // First line, get lang names
  ClipboardColumnsToLocIndex(sl[0], locIndex);

  SetLength(txt, fLocales.Count);

  // Other lines, get the strings
  filesCount := 0;
  changesCount := 0;

  // Line 0 - Column names
  // Line 1 - Translation percentage
  for I := 2 to sl.Count - 1 do
  if StartsStr('>', sl[I]) then
  begin
    // Save previous
    if I > 2 then
      Save(aSortById);

    // Take 1st cell and trim the header
    fname := StringFromString(sl[I], #9, 0);
    fname := StringReplace(fname, '> ', '', [rfReplaceAll]);

    Load4(fname, []);

    Inc(filesCount);
  end else
  if StringFromString(sl[I], #9, 0) <> '-' then
  begin
    // Source data
    tag := StringFromString(sl[I], #9, 0);

    if tag = '' then Continue;

    for K := 0 to High(locIndex) do
    begin
      s := StringFromString(sl[I], #9, K+1);
      if StartsStr(#39'=', s) then
        s := LeftStr(s, Length(s) - 1);

      txt[K] := s;
    end;

    // Now we have all data to update

    idLine := fLines.IndexOfTag(tag);

    if idLine = -1 then
      Continue; // Tag was removed from the game

    for K := 0 to High(locIndex) do
    begin
      idLoc := locIndex[K];
      if fLines[idLine].Strings[idLoc] <> txt[K] then
      begin
        fLines[idLine].Strings[idLoc] := txt[K];
        fLines[idLine].LastChanged[idLoc] := dt;
        Inc(changesCount);
      end;
    end;
  end;

  Save(aSortById);

  ShowMessage(Format('%d files updated. %d lines updated', [filesCount, changesCount]));

  sl.Free;
end;


procedure TKMTextManager.ListMismatchingAll(aFolders: TStringList; aList: TStringList; aLocales: TByteSet);
var
  I, K: Integer;
  newFile: Boolean;

  procedure Append(aString: string);
  begin
    if newFile then
    begin
      if aList.Count > 0 then
        aList.Append('');

      aList.Append(aFolders[I] + ': ');
      newFile := False;
    end;

    aList.Append(aString);
  end;
begin
  for I := 0 to aFolders.Count - 1 do
  begin
    Load4(aFolders[I], aLocales);

    newFile := True;

    // Linebreaks
    for K := 0 to fLines.Count - 1 do
      if not fLines[K].CheckMatchingForCharCount(['|'], aLocales) then
        Append('  | in ' + fLines[K].Tag);

    // Formats
    for K := 0 to fLines.Count - 1 do
      if not fLines[K].CheckMatchingForCharCount(['%'], aLocales) then
        Append('  % in ' + fLines[K].Tag);

    // Colors
    for K := 0 to fLines.Count - 1 do
      if not fLines[K].CheckMatchingForCharCount(['[$'], aLocales) then
        Append('  [$ in ' + fLines[K].Tag);
  end;
end;


end.