diff --git a/app/src/main/java/de/twonirwana/infinity/CsvPrinter.java b/app/src/main/java/de/twonirwana/infinity/CsvPrinter.java deleted file mode 100644 index 51c97b7..0000000 --- a/app/src/main/java/de/twonirwana/infinity/CsvPrinter.java +++ /dev/null @@ -1,104 +0,0 @@ -package de.twonirwana.infinity; - -import com.google.common.base.Strings; -import de.twonirwana.infinity.unit.api.*; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; - -import java.io.FileWriter; -import java.io.IOException; -import java.io.Writer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class CsvPrinter { - - public static void printAll(Database database) { - database.getAllSectorials().forEach(id -> printSectorialList(id, database.getAllUnitsForSectorialWithoutMercs(id))); - } - - public static void printSectorialList(Sectorial faction, List printableUnits) { - - - String[] headers = {"Name", "Profile Name", - "MOV", "CC", "BS", "PH", "WIP", "ARM", "BTS", "Wounds", "Silhouette", "AVA", - "Points", "SWC", - "Skills", "Equipment", "Weapons"}; - try { - Files.createDirectories(Path.of("out/csv/")); - } catch (IOException e) { - throw new RuntimeException(e); - } - - try (Writer writer = new FileWriter("out/csv/" + faction.getSlug() + ".csv"); - CSVPrinter csvPrinter = new CSVPrinter(writer, - CSVFormat.Builder.create().setDelimiter(';').setHeader(headers).get())) { - - printableUnits.stream() - .sorted(Comparator.comparing(UnitOption::getCombinedId)) - .filter(u -> !u.isMerc()) - .forEach(unitOption -> { - Trooper primaryUnit = unitOption.getPrimaryUnit(); - //todo support mutli profile units - String skills = Optional.ofNullable(primaryUnit.getProfiles().getFirst().getSkills()).stream() - .flatMap(Collection::stream) - .filter(Objects::nonNull) - .map(Skill::getName) - .filter(s -> !Strings.isNullOrEmpty(s)) - .collect(Collectors.joining(", ")); - String equipment = Optional.ofNullable(primaryUnit.getProfiles().getFirst().getEquipment()).stream() - .flatMap(Collection::stream) - .filter(Objects::nonNull) - .map(Equipment::getName) - .filter(s -> !Strings.isNullOrEmpty(s)) - .collect(Collectors.joining(", ")); - String weapons = Optional.ofNullable(primaryUnit.getProfiles().getFirst().getWeapons()).stream() - .flatMap(Collection::stream) - .filter(Objects::nonNull) - .map(Weapon::getName) - .filter(s -> !Strings.isNullOrEmpty(s)) - .distinct() - .collect(Collectors.joining(", ")); - - try { - csvPrinter.printRecord( - primaryUnit.getOptionName(), - primaryUnit.getProfiles().getFirst().getName(), - primaryUnit.getProfiles().getFirst().getMovementInCm().stream() - .map(DistanceUtil::toInch) - .map(Objects::toString) - .collect(Collectors.joining("-")), - primaryUnit.getProfiles().getFirst().getCloseCombat(), - primaryUnit.getProfiles().getFirst().getBallisticSkill(), - primaryUnit.getProfiles().getFirst().getPhysique(), - primaryUnit.getProfiles().getFirst().getWillpower(), - primaryUnit.getProfiles().getFirst().getArmor(), - primaryUnit.getProfiles().getFirst().getBioTechnologicalShield(), - primaryUnit.getProfiles().getFirst().getWounds(), - primaryUnit.getProfiles().getFirst().getSilhouette(), - primaryUnit.getProfiles().getFirst().getAvailability(), - primaryUnit.getCost(), - primaryUnit.getSpecialWeaponCost(), - skills, - equipment, - weapons - - //todo peripheral, notes, Structure vs Vital - ); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - csvPrinter.flush(); - - - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/app/src/main/java/de/twonirwana/infinity/ExportAll.java b/app/src/main/java/de/twonirwana/infinity/ExportAll.java index 6c7859a..9ce3d5e 100644 --- a/app/src/main/java/de/twonirwana/infinity/ExportAll.java +++ b/app/src/main/java/de/twonirwana/infinity/ExportAll.java @@ -1,18 +1,17 @@ package de.twonirwana.infinity; -import java.io.IOException; import java.time.LocalDateTime; public class ExportAll { - public static void main(String[] args) throws IOException { + static void main() { + // Database db = DatabaseImp.createWithoutUpdate("resources"); Database db = DatabaseImp.createTimedUpdate(); - CsvPrinter.printAll(db); HtmlPrinter htmlPrinter = new HtmlPrinter(LocalDateTime::now); - htmlPrinter.printAll(db, true, HtmlPrinter.Template.a7_image); + // htmlPrinter.printAll(db, true, HtmlPrinter.Template.a7_image); } diff --git a/data/core/build.gradle.kts b/data/core/build.gradle.kts index 8df2eb9..fb88a32 100644 --- a/data/core/build.gradle.kts +++ b/data/core/build.gradle.kts @@ -18,6 +18,7 @@ dependencies { implementation("io.micrometer:micrometer-core:1.16.5") implementation("io.avaje:avaje-config:5.1") implementation("io.avaje:avaje-applog-slf4j:1.2") + implementation("org.apache.commons:commons-csv:1.14.1") implementation("tools.jackson.core:jackson-core:3.1.3") implementation("com.fasterxml.jackson.core:jackson-annotations:2.21") diff --git a/data/core/src/main/java/de/twonirwana/infinity/CsvPrinter.java b/data/core/src/main/java/de/twonirwana/infinity/CsvPrinter.java new file mode 100644 index 0000000..9d3cadd --- /dev/null +++ b/data/core/src/main/java/de/twonirwana/infinity/CsvPrinter.java @@ -0,0 +1,149 @@ +package de.twonirwana.infinity; + +import de.twonirwana.infinity.unit.api.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class CsvPrinter { + + private static final Set WEAPON_TYPES = Set.of(Weapon.Type.WEAPON, Weapon.Type.TURRET); + + public static void printList(String name, List printableUnits) { + + String[] headers = { + "Sectorial", "ID", "Unit Name", "Option Name", "Unit Option Name", "Profile Name", + "MOV", "CC", "BS", "PH", "WIP", "ARM", "BTS", "Wounds", "Silhouette", "Orders", "AVA", + "Points", "SWC", + "Skills", "Equipment", "Weapons", "Characteristics", + "CB Image" + }; + + try { + Files.createDirectories(Path.of("out/csv/")); + } catch (IOException e) { + throw new RuntimeException(e); + } + + try (Writer writer = new FileWriter("out/csv/" + name + ".csv"); + CSVPrinter csvPrinter = new CSVPrinter(writer, + CSVFormat.Builder.create().setDelimiter(';').setHeader(headers).get())) { + + printableUnits.stream() + .sorted(Comparator.comparing(UnitOption::getCombinedId)) + .filter(u -> !u.isMerc()) + .forEach(unitOption -> unitOption.getAllTrooper() + .forEach(trooper -> trooper.getProfiles() + .forEach(profile -> printUnitOptionProfile(csvPrinter, unitOption, trooper, profile)))); + + csvPrinter.flush(); + log.info("Update of {} units have been printed into {}", printableUnits.size(), name); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static String prettyExtra(ExtraValue extraValue) { + if (extraValue.getType() == ExtraValue.Type.Text) { + return extraValue.getText().replace("UPGRADE: ", ""); + } else if (extraValue.getType() == ExtraValue.Type.Distance) { + String operator = extraValue.getDistanceCm() > 0 ? "+" : ""; + return "%s%s%s".formatted(operator, + DistanceUtil.convertString(extraValue.getDistanceCm(), true), + "″"); + } + throw new RuntimeException("Type not implemented"); + } + + private static void printUnitOptionProfile(CSVPrinter csvPrinter, UnitOption unitOption, Trooper trooper, TrooperProfile profile) { + String skills = profile.getSkills().stream() + .map(CsvPrinter::getSkillNameAndExtra) + .collect(Collectors.joining(", ")); + String equipment = profile.getEquipment().stream() + .map(CsvPrinter::getEquipmentNameAndExtra) + .collect(Collectors.joining(", ")); + String weapons = profile.getWeapons().stream() + .filter(w -> !"Suppressive Fire Mode Weapon".equals(w.getName())) + .filter(w -> WEAPON_TYPES.contains(w.getType())) + .map(CsvPrinter::getWeaponNameAndExtra) + .distinct() + .collect(Collectors.joining(", ")); + try { + + + csvPrinter.printRecord( + unitOption.getSectorial().getName(), + profile.getCombinedProfileId(), + unitOption.getUnitName(), + trooper.getOptionName(), + unitOption.getUnitOptionName(), + profile.getName(), + profile.getMovementInCm().stream() + .map(DistanceUtil::toInch) + .map(Objects::toString) + .collect(Collectors.joining("-")), + profile.getCloseCombat(), + profile.getBallisticSkill(), + profile.getPhysique(), + profile.getWillpower(), + profile.getArmor(), + profile.getBioTechnologicalShield(), + profile.getWounds(), + profile.getSilhouette(), + profile.getOrders().stream() + .map(o -> "%s[%d]".formatted(o.getType(), o.getTotal())) + .sorted() + .collect(Collectors.joining(", ")), + profile.getAvailability(), + unitOption.getTotalCost(), + unitOption.getTotalSpecialWeaponCost(), + skills, + equipment, + weapons, + String.join(", ", profile.getCharacteristics()), + !profile.getImageNames().isEmpty() + + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static String getSkillNameAndExtra(Skill skill) { + String extraString = skill.getExtras().isEmpty() ? "" : " [%s]".formatted(skill.getExtras().stream() + .map(CsvPrinter::prettyExtra) + .collect(Collectors.joining(", "))); + return "%s%s".formatted(skill.getName(), extraString); + } + + private static String getEquipmentNameAndExtra(Equipment equipment) { + String extraString = equipment.getExtras().isEmpty() ? "" : " [%s]".formatted(equipment.getExtras().stream() + .map(CsvPrinter::prettyExtra) + .collect(Collectors.joining(", "))); + return "%s%s".formatted(equipment.getName(), extraString); + } + + private static String getWeaponNameAndExtra(Weapon weapon) { + String extraString = weapon.getExtras().isEmpty() ? "" : " [%s]".formatted(weapon.getExtras().stream() + .map(CsvPrinter::prettyExtra) + .collect(Collectors.joining(", "))); + return "%s%s".formatted(weapon.getName(), extraString); + } + + private record UnitOptionTrooperProfile(UnitOption unitOption, Trooper trooper, TrooperProfile profile) { + } + +} diff --git a/data/core/src/main/java/de/twonirwana/infinity/db/DataLoader.java b/data/core/src/main/java/de/twonirwana/infinity/db/DataLoader.java index 7bc8f31..3602ef5 100644 --- a/data/core/src/main/java/de/twonirwana/infinity/db/DataLoader.java +++ b/data/core/src/main/java/de/twonirwana/infinity/db/DataLoader.java @@ -121,22 +121,29 @@ public DataLoader(UpdateOption updateOption, String resourcesFolder) throws IOEx if (updateNow) { log.info("update all files"); } - Metadata metadata = loadMetadata(updateNow); + MetadataAndUpdateFlag metadataAndUpdateFlag = loadMetadata(updateNow); + Metadata metadata = metadataAndUpdateFlag.metadata(); + boolean hasMetadataUpdate = metadataAndUpdateFlag.hasUpdate(); sectorialIdMap = metadata.getFactions().stream() .sorted(Comparator.comparingInt(Faction::getId)) .filter(f -> f.getId() != 901) // NA2 doesn't have a vanilla option .map(f -> new Sectorial(f.getId(), f.getParent(), - f.getName(), + f.getName().replace("\n", " "), f.getSlug(), f.isDiscontinued(), Utils.getFileNameFromUrl(f.getLogo()))) .collect(Collectors.toMap(Sectorial::getId, Function.identity())); - Map sectorialListMap = sectorialIdMap.values().stream() - .collect(Collectors.toMap(Function.identity(), f -> loadSectorial(f.getId(), f.getSlug(), updateNow))); + Map sectorialListAndUpdateFlags = sectorialIdMap.values().stream() + .collect(Collectors.toMap(Function.identity(), s -> loadSectorial(s.getId(), s.getSlug(), updateNow))); - Map reenforcementListMap = sectorialListMap.entrySet().stream() + boolean hasSectorialUpdate = sectorialListAndUpdateFlags.values().stream().anyMatch(SectorialListAndUpdateFlag::hasUpdate); + + Map sectorialListMap = sectorialListAndUpdateFlags.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, s -> s.getValue().sectorialList())); + + Map reenforcementUpdateListMap = sectorialListMap.entrySet().stream() .flatMap(e -> { if (e.getValue().getReinforcements() != null) { return Stream.of(e); @@ -152,9 +159,15 @@ public DataLoader(UpdateOption updateOption, String resourcesFolder) throws IOEx .collect(Collectors.toMap(Map.Entry::getKey, e -> loadSectorial(e.getValue().getReinforcements(), e.getKey().getSlug() + "_ref", updateNow))); - //todo ref image - sectorialIdMap.values() - .forEach(s -> downloadImageDataFile(s, updateNow)); + boolean hasSectorialRefUpdate = reenforcementUpdateListMap.values().stream().anyMatch(SectorialListAndUpdateFlag::hasUpdate); + Map reenforcementListMap = reenforcementUpdateListMap.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, s -> s.getValue().sectorialList())); + + boolean hasImageUpdate = sectorialIdMap.values() + .stream() + .anyMatch(s -> downloadImageDataFile(s, updateNow)); + + Map sectorialImageMap = sectorialIdMap.values().stream() .filter(s -> Paths.get(imageDataFileFormat.formatted(s.getId(), s.getSlug())).toFile().exists()) .collect(Collectors.toMap(Function.identity(), s -> deserializeSectorialImage(Paths.get(imageDataFileFormat.formatted(s.getId(), s.getSlug()))))); @@ -180,6 +193,19 @@ public DataLoader(UpdateOption updateOption, String resourcesFolder) throws IOEx metaChemistry = mapChemistryRolls(metadata); sectorialFireteamCharts = mapFireteamChat(sectorialListMap); + + if (hasMetadataUpdate || hasSectorialUpdate || hasSectorialRefUpdate || hasImageUpdate) { + List unitOptions = getAllUnits().stream() + .filter(u -> !u.getSectorial().isDiscontinued()) + .filter(u -> !u.isMerc()) + .filter(u -> !u.isReinforcementUnit()) + .distinct() + .sorted(Comparator.comparing(UnitOption::getCombinedId)) + .toList(); + + CsvPrinter.printList(DATE_TIME_FORMATTER.format(LocalDateTime.now()) + "_" + unitOptions.hashCode(), unitOptions); + } + //todo ref image } private static List getLastModifiedDates(String folderPath) throws IOException { @@ -313,7 +339,8 @@ private static String getBaseName(String pathString) { } //gson has the better pretty print format - private static void savePrettyJson(BufferedInputStream inputStream, Path targetFilePath) throws IOException { + private static boolean savePrettyJson(BufferedInputStream inputStream, Path targetFilePath) throws IOException { + boolean hasUpdate = false; JsonElement jsonElement; String baseFileName = getBaseName(targetFilePath.getFileName().toString()); try (InputStreamReader reader = new InputStreamReader(inputStream)) { @@ -343,6 +370,7 @@ private static void savePrettyJson(BufferedInputStream inputStream, Path targetF List diffs = JsonDiff.getDiffs(existingFile, newJson, List.of("resume", "filters", "peripheral")); updateCounter.increment(diffs.size()); + hasUpdate = true; diffs.forEach(diff -> log.info("{}.{}", baseFileName, diff.toString())); } else { @@ -356,6 +384,7 @@ private static void savePrettyJson(BufferedInputStream inputStream, Path targetF } Files.copy(tempFile, targetFilePath, StandardCopyOption.REPLACE_EXISTING); tempFile.toFile().delete(); + return hasUpdate; } private static SectorialImage deserializeSectorialImage(Path path) { @@ -388,49 +417,53 @@ private static Optional getStreamForURL(String urlString) { } } - private void downloadImageDataFile(Sectorial sectorial, boolean forceUpdate) { + private boolean downloadImageDataFile(Sectorial sectorial, boolean forceUpdate) { + boolean hasUpdate = false; createFolderIfNotExists(imageDataFolder); Path path = Paths.get(imageDataFileFormat.formatted(sectorial.getId(), sectorial.getSlug())); if (!path.toFile().exists() || forceUpdate) { try { Optional in = getStreamForURL(UNIT_IMAGE_URL.formatted(sectorial.getId())); if (in.isPresent()) { - savePrettyJson(in.get(), path); + hasUpdate = savePrettyJson(in.get(), path); } } catch (IOException e) { throw new RuntimeException(e); } } + return hasUpdate; } - private SectorialList loadSectorial(int id, String name, boolean forceUpdate) { + private SectorialListAndUpdateFlag loadSectorial(int id, String name, boolean forceUpdate) { + boolean hasUpdate = false; createFolderIfNotExists(sectorialFolder); Path path = Paths.get(sectorialFolder, SECTORIAL_FILE_FORMAT.formatted(id, name)); if (!path.toFile().exists() || forceUpdate) { try { Optional in = getStreamForURL(FACTION_URL_FORMAT.formatted(id)); if (in.isPresent()) { - savePrettyJson(in.get(), path); + hasUpdate = savePrettyJson(in.get(), path); } } catch (IOException e) { throw new RuntimeException(e); } } - return deserializeSectorialList(path); + return new SectorialListAndUpdateFlag(deserializeSectorialList(path), hasUpdate); } - private Metadata loadMetadata(boolean forceUpdate) throws IOException { + private MetadataAndUpdateFlag loadMetadata(boolean forceUpdate) throws IOException { + boolean hasUpdate = false; createFolderIfNotExists(resourcesFolder); Path path = Paths.get(metaDataFilePath); if (!path.toFile().exists() || forceUpdate) { Optional metaDataInput = getStreamForURL(META_DATA_URL); if (metaDataInput.isPresent()) { - savePrettyJson(metaDataInput.get(), path); + hasUpdate = savePrettyJson(metaDataInput.get(), path); } } - return objectMapper.readValue(path.toFile(), Metadata.class); + return new MetadataAndUpdateFlag(objectMapper.readValue(path.toFile(), Metadata.class), hasUpdate); } private void downloadAllSectorialLogos(Set logoUrls) { @@ -480,4 +513,10 @@ public enum UpdateOption { TIMED_UPDATE, NEVER_UPDATE } + + private record SectorialListAndUpdateFlag(SectorialList sectorialList, boolean hasUpdate) { + } + + private record MetadataAndUpdateFlag(Metadata metadata, boolean hasUpdate) { + } } diff --git a/data/core/src/main/java/de/twonirwana/infinity/db/UnitMapper.java b/data/core/src/main/java/de/twonirwana/infinity/db/UnitMapper.java index d96a8dc..e75d77f 100644 --- a/data/core/src/main/java/de/twonirwana/infinity/db/UnitMapper.java +++ b/data/core/src/main/java/de/twonirwana/infinity/db/UnitMapper.java @@ -289,7 +289,9 @@ private static List createAdditionalTroopers(List profi equipmentIdMap, sectorialImage, sectorialFilter)) - ).toList(); + ) + .sorted(Comparator.comparing(Trooper::getCombinedId)) + .toList(); } private static List getProfileGroupFromInclude(Unit unit, List profileIncludes) { @@ -602,6 +604,7 @@ private static List getUnitPeripheral(ProfileOption profileOption, Map peripheralFilter.get(pi.getId())) .filter(Objects::nonNull) + .sorted() .toList(); } @@ -647,8 +650,12 @@ private static TrooperProfile getOptionProfile(Sectorial sectorial, .filter(io -> io.getOptions().contains(profileOption.getId())) .map(ImgOption::getUrl) .map(Utils::getFileNameFromUrl) + .sorted() + .toList(); + List orders = profileOption.getOrders().stream() + .map(UnitMapper::mapOrder) + .sorted(Comparator.comparing(de.twonirwana.infinity.unit.api.Order::getType)) .toList(); - List orders = profileOption.getOrders().stream().map(UnitMapper::mapOrder).toList(); return new TrooperProfile( sectorial, @@ -713,6 +720,7 @@ private static Trooper createTrooper(Sectorial section, equipmentIdMap, sectorialImage, sectorialFilter)) + .sorted(Comparator.comparing(TrooperProfile::getProfileId)) .toList(); return new Trooper( section, @@ -737,6 +745,7 @@ private static List getUnitCharacteristics(ProfileOption profileOption, .filter(Objects::nonNull) .map(characteristicsFilter::get) .filter(Objects::nonNull) + .sorted() .toList(); }