Merge pull request #31 from anno-mods/devel/1.12.4

versioned bundle downloads, fixed modinfo defaults, improved texture caching

CHANGELOG.md

@@ -6,6 +6,8 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how
 
 ## 1.12 Simplified Build and Deploy
 
+- 1.12.4: Fixed issue with texture cache
+- 1.12.4: Include tag in downloaded bundle mods
 - 1.12.3: Allow array in `bundle`
 - 1.12.2: Export .png for icons as well
 - 1.12.1: Support hierarchy of named `Group`s (with comments) in outline

package.json

@@ -2,7 +2,7 @@
 	"name": "anno-modding-tools",
 	"displayName": "Anno Modding Tools",
 	"description": "Modding tools for Anno 1800",
-	"version": "1.12.3",
+	"version": "1.12.4",
 	"publisher": "JakobHarder",
 	"repository": {
 		"type": "git",

src/builder/ModCache.ts

@@ -30,13 +30,19 @@ export class ModCache {
     this._last = undefined;
   }
 
-  /** returns true if output is already available */
-  public use(sourceFile: string) {
+  /** Returns true if output is already available */
+  public use(sourceFile: string, targetPath?: string) {
     this._last = { sha: _getHash(sourceFile), output: {} };
 
+    const stillGood = (sha: string, targetFile: string) => {
+      return (sha === _getHash(targetFile)
+        && (!targetPath || targetFile.startsWith(targetPath)));
+    }
+
     // identical file in old cache means we can skip if old output is still correct
     if (this.old?.files && this.old.files[sourceFile]?.sha === this._last.sha) {
-      if (-1 === Object.entries(this.old.files[sourceFile].output).findIndex((e: any) => e[1].sha !== _getHash(e[0]))) {
+      const outputFiles = Object.entries(this.old.files[sourceFile].output);
+      if (-1 === outputFiles.findIndex((e: any) => !stillGood(e[1].sha, e[0]))) {
         // store old cache values
         this.data.files[sourceFile] = { ...this._last, ...this.old.files[sourceFile] };
         return true;
@@ -53,8 +59,10 @@ export class ModCache {
   }
 
   public output(targetFile: string) {
-    if (!this._last) { return; };
-
+    if (!this._last) {
+      return;
+    };
+
     this._last.output[targetFile] = { sha: _getHash(targetFile) };
   }
 

src/builder/converter/texture.ts

@@ -23,18 +23,11 @@ export class TextureConverter extends Converter {
       const lodLevels = Math.max(0, Math.min(9, options.converterOptions.lods === undefined ? 3 : options.converterOptions.lods));
       const changePath = options.converterOptions.changePath || '';
       const sourceFile = path.join(sourceFolder, file);
-      if (cache.use(sourceFile)) {
-        this._logger.log(`     no update required`);
-        continue;
-      }
 
       try {
         const dirname = path.dirname(file);
         const basename = path.basename(file, '.png');
-
         const mapsPath = (path.basename(dirname) === changePath) ? dirname : path.join(dirname, changePath);
-        utils.ensureDir(path.join(outFolder, mapsPath));
-        utils.ensureDir(path.join(options.cache, dirname));
 
         const lodFilePaths = [];
         if (lodLevels === 0) {
@@ -48,6 +41,14 @@ export class TextureConverter extends Converter {
         }
 
         const targetFolder = path.dirname(lodFilePaths[0]);
+        if (cache.use(sourceFile, targetFolder)) {
+          this._logger.log(`     no update required`);
+          continue;
+        }
+
+        utils.ensureDir(path.join(outFolder, mapsPath));
+        utils.ensureDir(path.join(options.cache, dirname));
+
         const textures = dds.convertToTexture(sourceFile, targetFolder, dds.TextureFormat.unknown, lodLevels);
         if (!textures) {
           return false;

src/builder/index.ts

@@ -50,11 +50,10 @@ export class ModBuilder {
 
   public async build(filePath: string): Promise<boolean> {
     this._logger.log('Build ' + filePath);
-    const modJson = JSON.parse(fs.readFileSync(filePath, 'utf8'));
-
-    // fill defaults
-    modJson.out = modJson.out ?? "${annoMods}/${modName}";
-    modJson.src = modJson.src ?? ".";
+    const modJson = utils.readModinfo(path.dirname(filePath));
+    if (!modJson) {
+      return false;
+    }
 
     let sourceFolders: string[] = Array.isArray(modJson.src) ? modJson.src : [ modJson.src ];
     sourceFolders = sourceFolders.map(x => path.dirname(filePath) + '/' + x);
@@ -157,17 +156,16 @@ export class ModBuilder {
     this._logger.log(`bundles`);
 
     if (modJson.bundle) {
-      const bundles = Array.isArray(modJson.bundle) ? modJson.bundle : Object.values(modJson.bundle);
-
-      for (const bundle of bundles) {
+      for (const bundle of modJson.bundle) {
         const fileName = path.basename(bundle);
-        const targetPath = path.join(cache, 'downloads', fileName);
+        const version = path.basename(path.dirname(bundle)).replace(/[^\w\-]/g, '');
+        const targetPath = path.join(cache, 'downloads', utils.insertEnding(fileName, '-' + version));
         if (!fs.existsSync(targetPath)) {
-          this._logger.log(`   * download ${fileName}`);
+          this._logger.log(`   * download ${version}/${fileName}`);
           utils.downloadFile(bundle, targetPath, this._logger);
         }
         else {
-          this._logger.log(`   * skip download of ${fileName}`);
+          this._logger.log(`   * skip download of ${version}/${fileName}`);
         }
         this._logger.log(`  <= extract content`);
         utils.extractZip(targetPath, outFolder, this._logger);

src/other/modFolder.ts

@@ -13,7 +13,7 @@ export namespace ModFolder {
     modsFolder_ = modsFolder;
 
     const modinfos = glob.sync('{,*/,*/*/}modinfo.json', { cwd: modsFolder, nodir: true });
-    
+
     for (const modinfoPath of modinfos) {
       try {
         const modinfo = JSON.parse(fs.readFileSync(path.join(modsFolder, modinfoPath), 'utf8'));

src/other/utils.ts

@@ -130,8 +130,12 @@ export function getAssetsXmlPath(modPath: string) {
   return filePath;
 }
 
-export interface IAnnomod {
-  modinfo?: any
+interface IModinfo {
+  out?: string
+  src: string | string[]
+  bundle?: string[]
+  modinfo: any
+  converter?: any
   getRequiredLoadAfterIds: (modinfo: any) => string[]
 }
 
@@ -145,11 +149,12 @@ export function getRequiredLoadAfterIds(modinfo: any): string[] {
   return dependencies.filter(dep => loadAfterIds.includes(dep));
 }
 
-export function readModinfo(modPath: string): any {
-  let result;
+export function readModinfo(modPath: string): IModinfo | undefined {
+  let result: IModinfo;
   try {
     if (fs.existsSync(path.join(modPath, 'modinfo.json'))) {
       result = {
+        ...JSON.parse(fs.readFileSync(path.join(modPath, 'modinfo.json'), 'utf8')),
         'modinfo': JSON.parse(fs.readFileSync(path.join(modPath, 'modinfo.json'), 'utf8')),
         getRequiredLoadAfterIds
       };
@@ -161,13 +166,32 @@ export function readModinfo(modPath: string): any {
         getRequiredLoadAfterIds
       };
     }
+    else {
+      return undefined;
+    }
   }
   catch {
     // mod jsons can be invalid pretty fast
     return undefined;
   }
 
+  result.out = result.out ?? "${annoMods}/${modName}";
   result.src = result.src ?? ".";
+
+  // convert url ModDependencies to bundle
+  result.bundle = result.bundle ?? [];
+  if (!Array.isArray(result.bundle)) {
+    result.bundle = Object.values(result.bundle);
+  }
+  if (result.modinfo.ModDependencies && Array.isArray(result.modinfo.ModDependencies)) {
+    for (let i = 0; i < result.modinfo?.ModDependencies.length; i++) {
+      if (result.modinfo.ModDependencies[i].startsWith("http")) {
+        result.bundle.push(result.modinfo.ModDependencies[i]);
+        result.modinfo.ModDependencies[i] = path.basename(result.modinfo.ModDependencies[i], '.zip');
+      }
+    }
+  }
+
   return result;
 }
 

src/other/xmltest.ts

@@ -2,7 +2,6 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as child from 'child_process';
 import * as glob from 'glob';
-import * as vscode from 'vscode';
 import * as logger from './logger';
 import * as utils from '../other/utils';
 import { ModFolder } from './modFolder';
@@ -99,10 +98,10 @@ export interface IIssue {
   group?: boolean
 }
 
-export function fetchIssues(vanillaXml: string, modPath: string, mainPatchFile: string, 
-    patchFile: string, patchContent: string, modsFolder: string | undefined, 
+export function fetchIssues(vanillaXml: string, modPath: string, mainPatchFile: string,
+    patchFile: string, patchContent: string, modsFolder: string | undefined,
     asAbsolutePath: (relative: string) => string): IIssue[] {
-  
+
   const removeNulls = <S>(value: S | undefined): value is S => value !== null;
   const tester = asAbsolutePath("./external/xmltest.exe");
   patchFile = patchFile.replace(/\\/g, '/');
@@ -111,11 +110,10 @@ export function fetchIssues(vanillaXml: string, modPath: string, mainPatchFile:
   try {
     const roots = utils.findModRoots(mainPatchFile).map(e => ['-m', e]);
     const annomod = utils.readModinfo(modPath);
-    let prepatch = annomod?.getRequiredLoadAfterIds(annomod?.modinfo) ?? [];
+    let prepatch = annomod?.getRequiredLoadAfterIds(annomod?.modinfo).map(e => ['-p', e]) ?? [];
 
     if (prepatch && modsFolder) {
-      prepatch = prepatch.map((e: string) => ModFolder.getModFolder(modsFolder, e) ?? "").filter((e: string) => e !== "");
-      prepatch = prepatch.map((e: string) => ['-p', e]);
+      prepatch = prepatch.map((e: string[]) => [ e[0], ModFolder.getModFolder(modsFolder, e[1]) ?? "" ]).filter((e: string[]) => e[1] && e[1] !== "");
     }
 
     const maxBuffer = 20;
@@ -148,8 +146,8 @@ export function fetchIssues(vanillaXml: string, modPath: string, mainPatchFile:
   const filteredLines = testerLines
     .filter(e => e.indexOf(patchFile) >= 0)
     .filter(e => {
-      return e.startsWith('w', levelIndex + 1) 
-        || e.startsWith('e', levelIndex + 1) 
+      return e.startsWith('w', levelIndex + 1)
+        || e.startsWith('e', levelIndex + 1)
         || e.startsWith('[debug] Time:', levelIndex);
     });