Added Index Processor for Mojang Indices.

Also added test for the mojang processor.
TBD: Progress System for Validations.
TBD: Processor for Distribution.json.

src/main/asset/model/engine/Asset.ts

@@ -0,0 +1,7 @@
+export interface Asset {
+    id: string
+    hash: string
+    size: number
+    url: string
+    path: string
+}

src/main/asset/model/engine/AssetGuardError.ts

@@ -0,0 +1,33 @@
+export class AssetGuardError extends Error {
+
+    code?: string
+    stack!: string
+    error?: Partial<Error & {code?: string;}>
+
+    constructor(message: string, error?: Partial<Error & {code?: string;}>) {
+        super(message)
+
+        Error.captureStackTrace(this, this.constructor)
+
+        // Reference: https://github.com/sindresorhus/got/blob/master/source/core/index.ts#L340
+        if(error) {
+
+            this.error = error
+            this.code = error?.code
+
+            if (error.stack != null) {
+                const indexOfMessage = this.stack.indexOf(this.message) + this.message.length;
+                const thisStackTrace = this.stack.slice(indexOfMessage).split('\n').reverse();
+                const errorStackTrace = error.stack.slice(error.stack.indexOf(error.message!) + error.message!.length).split('\n').reverse();
+    
+                // Remove duplicated traces
+                while (errorStackTrace.length !== 0 && errorStackTrace[0] === thisStackTrace[0]) {
+                    thisStackTrace.shift();
+                }
+    
+                this.stack = `${this.stack.slice(0, indexOfMessage)}${thisStackTrace.reverse().join('\n')}${errorStackTrace.reverse().join('\n')}`;
+            }
+
+        }
+    }
+}

src/main/asset/model/engine/IndexProcessor.ts

@@ -0,0 +1,12 @@
+import { Asset } from './Asset'
+
+export abstract class IndexProcessor {
+
+    constructor(
+        protected commonDir: string
+    ) {}
+
+    abstract async init(): Promise<void>
+    abstract async validate(): Promise<{[category: string]: Asset[]}>
+
+}

src/main/asset/model/mojang/VersionJson.ts

@@ -23,7 +23,7 @@ export interface BaseArtifact {
 
 }
 
-interface LibraryArtifact extends BaseArtifact {
+export interface LibraryArtifact extends BaseArtifact {
 
     path: string
 

src/main/asset/model/mojang/VersionManifest.ts

@@ -0,0 +1,15 @@
+export interface MojangVersionManifest {
+
+    latest: {
+        release: string
+        snapshot: string
+    }
+    versions: {
+        id: string
+        type: string
+        url: string
+        time: string
+        releaseTime: string
+    }[]
+
+}

src/main/asset/processor/MojangIndexProcessor.ts

@@ -0,0 +1,305 @@
+import { IndexProcessor } from '../model/engine/IndexProcessor'
+import got, { HTTPError, GotError, RequestError, ParseError, TimeoutError } from 'got'
+import { LoggerUtil } from '../../logging/loggerutil'
+import { pathExists, readFile, ensureDir, writeFile, readJson } from 'fs-extra'
+import { MojangVersionManifest } from '../model/mojang/VersionManifest'
+import { calculateHash, getVersionJsonPath, validateLocalFile, getLibraryDir, getVersionJarPath } from '../../util/FileUtils'
+import { dirname, join } from 'path'
+import { VersionJson, AssetIndex, LibraryArtifact } from '../model/mojang/VersionJson'
+import { AssetGuardError } from '../model/engine/AssetGuardError'
+import { Asset } from '../model/engine/Asset'
+import { isLibraryCompatible, getMojangOS } from '../../util/MojangUtils'
+
+export class MojangIndexProcessor extends IndexProcessor {
+
+    public static readonly LAUNCHER_JSON_ENDPOINT = 'https://launchermeta.mojang.com/mc/launcher.json'
+    public static readonly VERSION_MANIFEST_ENDPOINT = 'https://launchermeta.mojang.com/mc/game/version_manifest.json'
+    public static readonly ASSET_RESOURCE_ENDPOINT = 'http://resources.download.minecraft.net'
+
+    private readonly logger = LoggerUtil.getLogger('MojangIndexProcessor')
+
+    private versionJson!: VersionJson
+    private assetIndex!: AssetIndex
+    private client = got.extend({
+        responseType: 'json'
+    })
+
+    private handleGotError<T>(operation: string, error: GotError, dataProvider: () => T): T {
+        if(error instanceof HTTPError) {
+            this.logger.error(`Error during ${operation} request (HTTP Response ${error.response.statusCode})`, error)
+            this.logger.debug('Response Details:')
+            this.logger.debug('Body:', error.response.body)
+            this.logger.debug('Headers:', error.response.headers)
+        } else if(error instanceof RequestError) {
+            this.logger.error(`${operation} request recieved no response (${error.code}).`, error)
+        } else if(error instanceof TimeoutError) {
+            this.logger.error(`${operation} request timed out (${error.timings.phases.total}ms).`)
+        } else if(error instanceof ParseError) {
+            this.logger.error(`${operation} request recieved unexepected body (Parse Error).`)
+        } else {
+            // CacheError, ReadError, MaxRedirectsError, UnsupportedProtocolError, CancelError
+            this.logger.error(`Error during ${operation} request.`, error)
+        }
+
+        return dataProvider()
+    }
+
+    private assetPath: string
+
+    constructor(commonDir: string, protected version: string) {
+        super(commonDir)
+        this.assetPath = join(commonDir, 'assets')
+    }
+
+    /**
+     * Download https://launchermeta.mojang.com/mc/game/version_manifest.json
+     *   Unable to download:
+     *     Proceed, check versions directory for target version
+     *       If version.json not present, fatal error.
+     *       If version.json present, load and use.
+     *   Able to download:
+     *     Download, use in memory only.
+     *     Locate target version entry.
+     *     Extract hash
+     *     Validate local exists and matches hash
+     *       Condition fails: download
+     *         Download fails: fatal error
+     *         Download succeeds: Save to disk, continue
+     *       Passes: load from file
+     * 
+     * Version JSON in memory
+     *   Extract assetIndex
+     *     Check that local exists and hash matches
+     *       if false, download
+     *         download fails: fatal error
+     *       if true: load from disk and use
+     * 
+     * complete init when 3 files are validated and loaded.
+     * 
+     */
+    public async init() {
+
+        const versionManifest = await this.loadVersionManifest()
+        this.versionJson = await this.loadVersionJson(this.version, versionManifest)
+        this.assetIndex = await this.loadAssetIndex(this.versionJson)
+
+    }
+
+    private async loadAssetIndex(versionJson: VersionJson): Promise<AssetIndex> {
+        const assetIndexPath = this.getAssetIndexPath(versionJson.assetIndex.id)
+        const assetIndex = await this.loadContentWithRemoteFallback<AssetIndex>(versionJson.assetIndex.url, assetIndexPath, { algo: 'sha1', value: versionJson.assetIndex.sha1 })
+        if(assetIndex == null) {
+            throw new AssetGuardError(`Failed to download ${versionJson.assetIndex.id} asset index.`)
+        }
+        return assetIndex
+    }
+
+    private async loadVersionJson(version: string, versionManifest: MojangVersionManifest | null): Promise<VersionJson> {
+        const versionJsonPath = getVersionJsonPath(this.commonDir, version)
+        if(versionManifest != null) {
+            const versionJsonUrl = this.getVersionJsonUrl(version, versionManifest)
+            if(versionJsonUrl == null) {
+                throw new AssetGuardError(`Invalid version: ${version}.`)
+            }
+            const hash = this.getVersionJsonHash(versionJsonUrl)
+            if(hash == null) {
+                throw new AssetGuardError(`Format of Mojang's version manifest has changed. Unable to proceed.`)
+            }
+            const versionJson = await this.loadContentWithRemoteFallback<VersionJson>(versionJsonUrl, versionJsonPath, { algo: 'sha1', value: hash })
+            if(versionJson == null) {
+                throw new AssetGuardError(`Failed to download ${version} json index.`)
+            }
+
+            return versionJson
+            
+        } else {
+            // Attempt to find local index.
+            if(await pathExists(versionJsonPath)) {
+                return await readJson(versionJsonPath)
+            } else {
+                throw new AssetGuardError(`Unable to load version manifest and ${version} json index does not exist locally.`)
+            }
+        }
+    }
+
+    private async loadContentWithRemoteFallback<T>(url: string, path: string, hash?: {algo: string, value: string}): Promise<T | null> {
+
+        try {
+            if(await pathExists(path)) {
+                const buf = await readFile(path)
+                if(hash) {
+                    const bufHash = calculateHash(buf, hash.algo)
+                    if(bufHash === hash.value) {
+                        return JSON.parse(buf.toString())
+                    }
+                } else {
+                    return JSON.parse(buf.toString())
+                }
+            }
+        } catch(error) {
+            throw new AssetGuardError(`Failure while loading ${path}.`, error)
+        }
+        
+        try {
+            const res = await this.client.get<T>(url)
+
+            await ensureDir(dirname(path))
+            await writeFile(path, res.body)
+
+            return res.body
+        } catch(error) {
+            return this.handleGotError(url, error as GotError, () => null)
+        }
+
+    }
+
+    private async loadVersionManifest(): Promise<MojangVersionManifest | null> {
+        try {
+            const res = await this.client.get<MojangVersionManifest>(MojangIndexProcessor.VERSION_MANIFEST_ENDPOINT)
+            return res.body
+        } catch(error) {
+            return this.handleGotError('Load Mojang Version Manifest', error as GotError, () => null)
+        }
+    }
+
+    private getVersionJsonUrl(id: string, manifest: MojangVersionManifest): string | null {
+        for(const version of manifest.versions) {
+            if(version.id == id){
+                return version.url
+            }
+        }
+        return null
+    }
+
+    private getVersionJsonHash(url: string): string | null {
+        const regex = /^https:\/\/launchermeta.mojang.com\/v1\/packages\/(.+)\/.+.json$/
+        const match = regex.exec(url)
+        if(match != null && match[1]) {
+            return match[1]
+        } else {
+            return null
+        }
+    }
+
+    private getAssetIndexPath(id: string): string {
+        return join(this.assetPath, 'indexes', `${id}.json`)
+    }
+
+    //  TODO progress tracker
+    public async validate() {
+
+        const assets = await this.validateAssets(this.assetIndex)
+        const libraries = await this.validateLibraries(this.versionJson)
+        const client = await this.validateClient(this.versionJson)
+        const logConfig = await this.validateLogConfig(this.versionJson)
+
+        return {
+            assets,
+            libraries,
+            client,
+            misc: [
+                ...logConfig
+            ]
+        }
+    }
+
+    private async validateAssets(assetIndex: AssetIndex): Promise<Asset[]> {
+
+        const objectDir = join(this.assetPath, 'objects')
+        const notValid: Asset[] = []
+
+        for(const assetEntry of Object.entries(assetIndex.objects)) {
+            const hash = assetEntry[1].hash
+            const path = join(objectDir, hash.substring(0, 2), hash)
+            const url = `${MojangIndexProcessor.ASSET_RESOURCE_ENDPOINT}/${hash.substring(0, 2)}/${hash}`
+
+            if(!await validateLocalFile(path, 'sha1', hash)) {
+                notValid.push({
+                    id: assetEntry[0],
+                    hash,
+                    size: assetEntry[1].size,
+                    url,
+                    path
+                })
+            }
+        }
+
+        return notValid
+
+    }
+
+    private async validateLibraries(versionJson: VersionJson): Promise<Asset[]> {
+        
+        const libDir = getLibraryDir(this.commonDir)
+        const notValid: Asset[] = []
+
+        for(const libEntry of versionJson.libraries) {
+            if(isLibraryCompatible(libEntry.rules, libEntry.natives)) {
+                let artifact: LibraryArtifact
+                if(libEntry.natives == null) {
+                    artifact = libEntry.downloads.artifact
+                } else {
+                    // @ts-ignore
+                    const classifier = libEntry.natives[getMojangOS()].replace('${arch}', process.arch.replace('x', ''))
+                    // @ts-ignore
+                    artifact = libEntry.downloads.classifiers[classifier]
+                }
+
+                const path = join(libDir, artifact.path)
+                const hash = artifact.sha1
+                if(!await validateLocalFile(path, 'sha1', hash)) {
+                    notValid.push({
+                        id: libEntry.name,
+                        hash,
+                        size: artifact.size,
+                        url: artifact.url,
+                        path
+                    })
+                }
+            }
+        }
+
+        return notValid
+    }
+
+    private async validateClient(versionJson: VersionJson): Promise<Asset[]> {
+
+        const version = versionJson.id
+        const versionJarPath = getVersionJarPath(this.commonDir, version)
+        const hash = versionJson.downloads.client.sha1
+
+        if(!await validateLocalFile(versionJarPath, 'sha1', hash)) {
+            return [{
+                id: `${version} client`,
+                hash,
+                size: versionJson.downloads.client.size,
+                url: versionJson.downloads.client.url,
+                path: versionJarPath
+            }]
+        }
+
+        return []
+
+    }
+
+    private async validateLogConfig(versionJson: VersionJson): Promise<Asset[]> {
+
+        const logFile = versionJson.logging.client.file
+        const path = join(this.assetPath, 'log_configs', logFile.id)
+        const hash = logFile.sha1
+
+        if(!await validateLocalFile(path, 'sha1', hash)) {
+            return [{
+                id: logFile.id,
+                hash,
+                size: logFile.size,
+                url: logFile.url,
+                path
+            }]
+        }
+
+        return []
+
+    }
+
+}

src/main/util/FileUtils.ts

@@ -0,0 +1,34 @@
+import { createHash } from 'crypto'
+import { join } from 'path'
+import { pathExists, readFile } from 'fs-extra'
+
+export function calculateHash(buf: Buffer, algo: string) {
+    return createHash(algo).update(buf).digest('hex')
+}
+
+export async function validateLocalFile(path: string, algo: string, hash?: string): Promise<boolean> {
+    if(await pathExists(path)) {
+        if(hash == null) {
+            return true
+        }
+        const buf = await readFile(path)
+        return calculateHash(buf, algo) === hash
+    }
+    return false
+}
+
+function getVersionExtPath(commonDir: string, version: string, ext: string) {
+    return join(commonDir, 'versions', version, `${version}.${ext}`)
+}
+
+export function getVersionJsonPath(commonDir: string, version: string) {
+    return getVersionExtPath(commonDir, version, 'json')
+}
+
+export function getVersionJarPath(commonDir: string, version: string) {
+    return getVersionExtPath(commonDir, version, 'jar')
+}
+
+export function getLibraryDir(commonDir: string) {
+    return join(commonDir, 'libraries')
+}

src/main/util/MojangUtils.ts

@@ -0,0 +1,60 @@
+import { Rule, Natives } from "../asset/model/mojang/VersionJson"
+
+export function getMojangOS(): string {
+    const opSys = process.platform
+    switch(opSys) {
+        case 'darwin':
+            return 'osx'
+        case 'win32':
+            return 'windows'
+        case 'linux':
+            return 'linux'
+        default:
+            return opSys
+    }
+}
+
+export function validateLibraryRules(rules?: Rule[]): boolean {
+    if(rules == null) {
+        return false
+    }
+    for(const rule of rules){
+        if(rule.action != null && rule.os != null){
+            const osName = rule.os.name
+            const osMoj = getMojangOS()
+            if(rule.action === 'allow'){
+                return osName === osMoj
+            } else if(rule.action === 'disallow'){
+                return osName !== osMoj
+            }
+        }
+    }
+    return true
+}
+
+export function validateLibraryNatives(natives?: Natives): boolean {
+    return natives == null ? true : Object.hasOwnProperty.call(natives, getMojangOS())
+}
+
+export function isLibraryCompatible(rules?: Rule[], natives?: Natives): boolean {
+    return rules == null ? validateLibraryNatives(natives) : validateLibraryRules(rules)
+}
+
+/**
+ * Returns true if the actual version is greater than
+ * or equal to the desired version.
+ * 
+ * @param {string} desired The desired version.
+ * @param {string} actual The actual version.
+ */
+export function mcVersionAtLeast(desired: string, actual: string){
+    const des = desired.split('.')
+    const act = actual.split('.')
+
+    for(let i=0; i<des.length; i++){
+        if(!(parseInt(act[i]) >= parseInt(des[i]))){
+            return false
+        }
+    }
+    return true
+}

test/assets/MojangIndexProcessorTest.ts

@@ -0,0 +1,132 @@
+import nock from 'nock'
+import { URL } from 'url'
+import { MojangIndexProcessor } from '../../src/main/asset/processor/MojangIndexProcessor'
+import { dirname, join } from 'path'
+import { expect } from 'chai'
+import { remove, pathExists } from 'fs-extra'
+import { getVersionJsonPath } from '../../src/main/util/FileUtils'
+
+// @ts-ignore (JSON Modules enabled in tsconfig.test.json)
+import versionManifest from './files/version_manifest.json'
+// @ts-ignore (JSON Modules enabled in tsconfig.test.json)
+import versionJson115 from './files/1.15.2.json'
+// @ts-ignore (JSON Modules enabled in tsconfig.test.json)
+import versionJson1710 from './files/1.7.10.json'
+// @ts-ignore (JSON Modules enabled in tsconfig.test.json)
+import index115 from './files/index_1.15.json'
+
+const commonDir = join(__dirname, 'files')
+const assetDir = join(commonDir, 'assets')
+const jsonPath115 = getVersionJsonPath(commonDir, '1.15.2')
+const indexPath115 = join(assetDir, 'indexes', '1.15.json')
+const jsonPath1710 = getVersionJsonPath(commonDir, '1.7.10')
+
+describe('Mojang Index Processor', () => {
+
+    after(async () => {
+        nock.cleanAll()
+        await remove(dirname(jsonPath115))
+        await remove(indexPath115)
+        await remove(dirname(jsonPath1710))
+    })
+
+    it('[ MIP ] Validate Full Remote (1.15.2)', async () => {
+
+        const manifestUrl = new URL(MojangIndexProcessor.VERSION_MANIFEST_ENDPOINT)
+        const versionJsonUrl = new URL('https://launchermeta.mojang.com/v1/packages/1a36ca2e147f4fdc4a8b9c371450e1581732c354/1.15.2.json')
+        const assetIndexUrl = new URL('https://launchermeta.mojang.com/v1/packages/5406d9a75dfb58f549070d8bae279562c38a68f6/1.15.json')
+
+        nock(manifestUrl.origin)
+            .get(manifestUrl.pathname)
+            .reply(200, versionManifest)
+
+        nock(versionJsonUrl.origin)
+            .get(versionJsonUrl.pathname)
+            .reply(200, versionJson115)
+
+        nock(assetIndexUrl.origin)
+            .get(assetIndexUrl.pathname)
+            .reply(200, index115)
+
+        const mojangIndexProcessor = new MojangIndexProcessor(commonDir, '1.15.2')
+        await mojangIndexProcessor.init()
+
+        const notValid = await mojangIndexProcessor.validate()
+
+        const savedJson = await pathExists(jsonPath115)
+        const savedIndex = await pathExists(indexPath115)
+
+        expect(notValid).to.haveOwnProperty('assets')
+        expect(notValid.assets).to.have.lengthOf(2109-2)
+        expect(notValid).to.haveOwnProperty('libraries')
+        // Natives are different per OS
+        expect(notValid.libraries).to.have.length.gte(24)
+        expect(notValid).to.haveOwnProperty('client')
+        expect(notValid.client).to.have.lengthOf(1)
+        expect(notValid).to.haveOwnProperty('misc')
+        expect(notValid.misc).to.have.lengthOf(1)
+
+        expect(savedJson).to.equal(true)
+        expect(savedIndex).to.equal(true)
+
+    })
+
+    it('[ MIP ] Validate Full Local (1.12.2)', async () => {
+
+        const manifestUrl = new URL(MojangIndexProcessor.VERSION_MANIFEST_ENDPOINT)
+
+        nock(manifestUrl.origin)
+            .get(manifestUrl.pathname)
+            .reply(200, versionManifest)
+
+        const mojangIndexProcessor = new MojangIndexProcessor(commonDir, '1.12.2')
+        await mojangIndexProcessor.init()
+
+        const notValid = await mojangIndexProcessor.validate()
+        expect(notValid).to.haveOwnProperty('assets')
+        expect(notValid.assets).to.have.lengthOf(1305-2)
+        expect(notValid).to.haveOwnProperty('libraries')
+        // Natives are different per OS
+        expect(notValid.libraries).to.have.length.gte(27)
+        expect(notValid).to.haveOwnProperty('client')
+        expect(notValid.client).to.have.lengthOf(1)
+        expect(notValid).to.haveOwnProperty('misc')
+        expect(notValid.misc).to.have.lengthOf(1)
+
+    })
+
+    it('[ MIP ] Validate Half Remote (1.7.10)', async () => {
+
+        const manifestUrl = new URL(MojangIndexProcessor.VERSION_MANIFEST_ENDPOINT)
+        const versionJsonUrl = new URL('https://launchermeta.mojang.com/v1/packages/2e818dc89e364c7efcfa54bec7e873c5f00b3840/1.7.10.json')
+
+        nock(manifestUrl.origin)
+            .get(manifestUrl.pathname)
+            .reply(200, versionManifest)
+
+        nock(versionJsonUrl.origin)
+            .get(versionJsonUrl.pathname)
+            .reply(200, versionJson1710)
+
+        const mojangIndexProcessor = new MojangIndexProcessor(commonDir, '1.7.10')
+        await mojangIndexProcessor.init()
+
+        const notValid = await mojangIndexProcessor.validate()
+
+        const savedJson = await pathExists(jsonPath1710)
+
+        expect(notValid).to.haveOwnProperty('assets')
+        expect(notValid.assets).to.have.lengthOf(686-2)
+        expect(notValid).to.haveOwnProperty('libraries')
+        // Natives are different per OS
+        expect(notValid.libraries).to.have.length.gte(27)
+        expect(notValid).to.haveOwnProperty('client')
+        expect(notValid.client).to.have.lengthOf(1)
+        expect(notValid).to.haveOwnProperty('misc')
+        expect(notValid.misc).to.have.lengthOf(1)
+
+        expect(savedJson).to.equal(true)
+
+    })
+
+})

test/assets/files/version_manifest.json

@@ -0,0 +1 @@
+{"latest":{"release":"1.15.2","snapshot":"20w16a"},"versions":[{"id":"1.15.2","type":"release","url":"https://launchermeta.mojang.com/v1/packages/1a36ca2e147f4fdc4a8b9c371450e1581732c354/1.15.2.json","time":"2020-04-15T13:24:27+00:00","releaseTime":"2020-01-17T10:03:52+00:00"},{"id":"1.12.2","type":"release","url":"https://launchermeta.mojang.com/v1/packages/6e69e85d0f85f4f4b9e12dd99d102092a6e15918/1.12.2.json","time":"2019-06-28T07:05:57+00:00","releaseTime":"2017-09-18T08:39:46+00:00"},{"id":"1.7.10","type":"release","url":"https://launchermeta.mojang.com/v1/packages/2e818dc89e364c7efcfa54bec7e873c5f00b3840/1.7.10.json","time":"2019-06-28T07:06:16+00:00","releaseTime":"2014-05-14T17:29:23+00:00"}]}

tsconfig.test.json

@@ -1,7 +1,8 @@
 {
   "compilerOptions": {
     "module": "commonjs",
-    "esModuleInterop": true
+    "esModuleInterop": true,
+    "resolveJsonModule": true
   },
   "extends": "./tsconfig.json"
 }