diff --git a/config.ini b/config.ini index a402d39..36c935e 100644 --- a/config.ini +++ b/config.ini @@ -1,18 +1,18 @@ -output.folder.furniture=/home/user/WebstormProjects/sites/assets.nitro.se/game/dcr/furniture/ -output.folder.figure=/home/user/WebstormProjects/sites/assets.nitro.se/game/gordon/figure/ +output.folder.furniture=/home/user/WebstormProjects/sites/assets.nitro.se/game/dcr/furniture-test/ +output.folder.figure=/home/user/WebstormProjects/sites/assets.nitro.se/game/gordon/figure-test/ output.folder.effect=/home/user/WebstormProjects/sites/assets.nitro.se/game/gordon/effect-test/ -output.folder.pet=/home/user/WebstormProjects/sites/assets.nitro.se/game/gordon/pet/ +output.folder.pet=/home/user/WebstormProjects/sites/assets.nitro.se/game/gordon/pet-test/ furnidata.url=http://assets.nitro.se/game/gamedata/furnidata-entry.xml figuremap.url=http://assets.nitro.se/game/gordon/PRODUCTION-201701242205-837386173/figuremap.xml effectmap.url=http://assets.nitro.se/game/gordon/PRODUCTION-201701242205-837386173/effectmap.xml external_vars.url=http://assets.nitro.se/game/gamedata/external_variables.txt -dynamic.download.url.furniture=/home/user/WebstormProjects/sites/assets.nitro.se/game/dcr/endrit/hof_furni/%className%.swf -dynamic.download.url.figure=/home/user/WebstormProjects/sites/assets.nitro.se/game/gordon/PRODUCTION-201701242205-837386173/%className%.swf -dynamic.download.url.effect=/home/user/WebstormProjects/sites/assets.nitro.se/game/gordon/PRODUCTION-201701242205-837386173/%className%.swf -dynamic.download.url.pet=/home/user/WebstormProjects/sites/assets.nitro.se/game/gordon/PRODUCTION-201701242205-837386173/%className%.swf -convert.furniture=1 +dynamic.download.url.furniture=http://assets.nitro.se/game/dcr/endrit/hof_furni/%className%.swf +dynamic.download.url.figure=http://assets.nitro.se/game/gordon/PRODUCTION-201701242205-837386173/%className%.swf +dynamic.download.url.effect=http://assets.nitro.se/game/gordon/PRODUCTION-201701242205-837386173/%className%.swf +dynamic.download.url.pet=http://assets.nitro.se/game/gordon/PRODUCTION-201701242205-837386173/%className%.swf +convert.furniture=0 convert.figure=0 -convert.effect=0 +convert.effect=1 convert.pet=0 figure.rotation.enabled=0 figure.skip.non-existing.asset.images=0 \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 133b491..a2780de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -678,18 +678,6 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" }, - "swf-extract": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/swf-extract/-/swf-extract-1.1.0.tgz", - "integrity": "sha1-DS6Q01lKFu9ly8hfuEEiRhdz9t0=", - "requires": { - "concat-frames": "^1.0.3", - "jpg-stream": "^1.1.1", - "lzma-purejs": "~0.9.3", - "png-stream": "^1.0.5", - "stream-to-array": "^2.3.0" - } - }, "tinify": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/tinify/-/tinify-1.5.0.tgz", diff --git a/package.json b/package.json index 16faf99..1767a6e 100644 --- a/package.json +++ b/package.json @@ -12,11 +12,14 @@ "@gizeta/swf-reader": "^1.0.0", "@types/node": "^14.14.22", "bytebuffer": "^5.0.1", + "concat-frames": "^1.0.3", "free-tex-packer-core": "^0.3.2", + "jpg-stream": "^1.1.2", "lodash": "^4.17.20", "node-fetch": "^2.6.1", "node-gzip": "^1.1.2", - "swf-extract": "^1.1.0", + "png-stream": "^1.0.5", + "stream-to-array": "^2.3.0", "xml2js": "^0.4.23" } } diff --git a/src/config/Configuration.ts b/src/config/Configuration.ts index dcc6dd7..879f855 100644 --- a/src/config/Configuration.ts +++ b/src/config/Configuration.ts @@ -10,7 +10,7 @@ export default class Configuration { } async init() { - const content = await fs.readFile("/home/user/git/nitro-asset-converter-node (copy)/config.ini"); + const content = await fs.readFile("/home/user/git/nitro-asset-converter-node/config.ini"); this.parseContent(content.toString("utf-8")); } diff --git a/src/downloaders/EffectDownloader.ts b/src/downloaders/EffectDownloader.ts index b224ed0..92c80c9 100644 --- a/src/downloaders/EffectDownloader.ts +++ b/src/downloaders/EffectDownloader.ts @@ -2,13 +2,9 @@ import Configuration from "../config/Configuration"; import HabboAssetSWF from "../swf/HabboAssetSWF"; import File from "../utils/File"; -const fs = require("fs"); const fetch = require('node-fetch'); const xml2js = require('xml2js'); const parser = new xml2js.Parser(/* options */); -const util = require('util'); - -const readFile = util.promisify(fs.readFile); export default class EffectDownloader { private readonly _config: Configuration; @@ -25,12 +21,11 @@ export default class EffectDownloader { const figureMap = await this.parseEffectMap(); const map = figureMap.map; + let count = 0; for (const lib of map.effect) { const info = lib['$']; const className: string = info.lib; - //if (className !== 'Hoverboard' && className !== 'Staff' && className !== 'ZombieMask' && className !== 'ESredUntouchable') continue; - const assetOutputFolder = new File(outputFolderEffect + "/" + className); if (assetOutputFolder.exists()) { continue; @@ -38,13 +33,27 @@ export default class EffectDownloader { if (!EffectDownloader.types.has(className)) { const url = this._config.getValue("dynamic.download.url.effect").replace("%className%", className); - if (!fs.existsSync(url)) { - console.log("SWF File does not exist: " + url); - continue; + let buffer: Buffer | null = null; + + if (url.includes("http")) { + const fetchData = await fetch(url); + if (fetchData.status === 404) { + console.log("SWF File does not exist: " + url); + continue; + } + + const arrayBuffer = await fetchData.arrayBuffer(); + buffer = Buffer.from(arrayBuffer); + } else { + const file = new File(url); + if (!file.exists()) { + console.log("SWF File does not exist: " + file.path); + return; + } } try { - const newHabboAssetSWF: HabboAssetSWF = new HabboAssetSWF(url); + const newHabboAssetSWF: HabboAssetSWF = new HabboAssetSWF(buffer !== null ? buffer : url); await newHabboAssetSWF.setupAsync(); EffectDownloader.types.set(className, info.type); diff --git a/src/downloaders/FigureDownloader.ts b/src/downloaders/FigureDownloader.ts index e7598ea..7daf9f9 100644 --- a/src/downloaders/FigureDownloader.ts +++ b/src/downloaders/FigureDownloader.ts @@ -6,9 +6,6 @@ const fs = require("fs"); const fetch = require('node-fetch'); const xml2js = require('xml2js'); const parser = new xml2js.Parser(/* options */); -const util = require('util'); - -const readFile = util.promisify(fs.readFile); export default class FigureDownloader { @@ -43,12 +40,26 @@ export default class FigureDownloader { className !== "jacket_U_snowwar4_team2") { //TODO: Figure out why snowstorm assets aren't converting... const url = this._config.getValue("dynamic.download.url.figure").replace("%className%", className); - if (!fs.existsSync(url)) { - console.log("SWF File does not exist: " + url); - return; + let buffer: Buffer | null = null; + + if (url.includes("http")) { + const fetchData = await fetch(url); + if (fetchData.status === 404) { + console.log("SWF File does not exist: " + url); + continue; + } + + const arrayBuffer = await fetchData.arrayBuffer(); + buffer = Buffer.from(arrayBuffer); + } else { + const file = new File(url); + if (!file.exists()) { + console.log("SWF File does not exist: " + file.path); + return; + } } - const newHabboAssetSWF: HabboAssetSWF = new HabboAssetSWF(url); + const newHabboAssetSWF: HabboAssetSWF = new HabboAssetSWF(buffer !== null ? buffer : url); await newHabboAssetSWF.setupAsync(); FigureDownloader.types.set(className, lib.part[0]['$'].type); diff --git a/src/downloaders/FurnitureDownloader.ts b/src/downloaders/FurnitureDownloader.ts index f3cfd1c..2e90014 100644 --- a/src/downloaders/FurnitureDownloader.ts +++ b/src/downloaders/FurnitureDownloader.ts @@ -29,12 +29,16 @@ export default class FurnitureDownloader { const roomitemtypes = furniData.roomitemtypes; const wallitemtypes = furniData.wallitemtypes; + const classNames: Array = new Array(); for (const roomItem of roomitemtypes) { for (const furnitype of roomItem.furnitype) { const attributes = furnitype['$']; const className = attributes.classname.split("\*")[0]; const revision = furnitype.revision[0]; + if (classNames.includes(className)) continue; + else classNames.push(className); + const assetOuputFolder = new File(outputFolderFurniture.path + "/" + className); if (assetOuputFolder.isDirectory()) { continue; @@ -49,6 +53,8 @@ export default class FurnitureDownloader { const attributes = furnitype['$']; const className = attributes.classname.split("\*")[0]; const revision = furnitype.revision[0]; + if (classNames.includes(className)) continue; + else classNames.push(className); const assetOuputFolder = new File(outputFolderFurniture + "/" + className); if (assetOuputFolder.isDirectory()) { @@ -66,14 +72,27 @@ export default class FurnitureDownloader { //if (className !== 'scifidoor') return; const url = this._config.getValue("dynamic.download.url.furniture").replace("%revision%", revision).replace("%className%", className); - const file = new File(url); - if (!file.exists()) { - console.log("SWF File does not exist: " + file.path); - return; + let buffer: Buffer | null = null; + + if (url.includes("http")) { + const fetchData = await fetch(url); + if (fetchData.status === 404) { + console.log("SWF File does not exist: " + url); + return; + } + + const arrayBuffer = await fetchData.arrayBuffer(); + buffer = Buffer.from(arrayBuffer); + } else { + const file = new File(url); + if (!file.exists()) { + console.log("SWF File does not exist: " + file.path); + return; + } } try { - const newHabboAssetSWF: HabboAssetSWF = new HabboAssetSWF(url); + const newHabboAssetSWF: HabboAssetSWF = new HabboAssetSWF(buffer !== null ? buffer : url); await newHabboAssetSWF.setupAsync(); await callback(newHabboAssetSWF, className); diff --git a/src/downloaders/PetDownloader.ts b/src/downloaders/PetDownloader.ts index 2c67569..13643b4 100644 --- a/src/downloaders/PetDownloader.ts +++ b/src/downloaders/PetDownloader.ts @@ -2,9 +2,7 @@ import Configuration from "../config/Configuration"; import HabboAssetSWF from "../swf/HabboAssetSWF"; import File from "../utils/File"; -const util = require('util'); -const fs = require("fs"); -const readFile = util.promisify(fs.readFile); +const fetch = require('node-fetch'); export default class PetDownloader { private readonly _config: Configuration; @@ -30,13 +28,26 @@ export default class PetDownloader { if (!itemClassNames.includes(pet)) { const url = this._config.getValue("dynamic.download.url.pet").replace("%className%", pet); - const file = new File(url); - if (!file.exists()) { - console.log("SWF File does not exist: " + file.path); - continue; + let buffer: Buffer | null = null; + + if (url.includes("http")) { + const fetchData = await fetch(url); + if (fetchData.status === 404) { + console.log("SWF File does not exist: " + url); + continue; + } + + const arrayBuffer = await fetchData.arrayBuffer(); + buffer = Buffer.from(arrayBuffer); + } else { + const file = new File(url); + if (!file.exists()) { + console.log("SWF File does not exist: " + file.path); + return; + } } - const newHabboAssetSWF: HabboAssetSWF = new HabboAssetSWF(url); + const newHabboAssetSWF: HabboAssetSWF = new HabboAssetSWF(buffer !== null ? buffer : url); await newHabboAssetSWF.setupAsync(); await callback(newHabboAssetSWF); diff --git a/src/swf/HabboAssetSWF.ts b/src/swf/HabboAssetSWF.ts index 6ca459c..b353908 100644 --- a/src/swf/HabboAssetSWF.ts +++ b/src/swf/HabboAssetSWF.ts @@ -12,13 +12,13 @@ export default class HabboAssetSWF { private _documentClass: string | null = null; constructor( - private readonly _path: string + private readonly _data: string | Buffer ) { this._tags = new Array(); } async setupAsync() { - const swf = await readSwfAsync(this._path); + const swf = await readSwfAsync(this._data); for (const tag of swf.tags) { switch (tag.header.code) { @@ -38,22 +38,22 @@ export default class HabboAssetSWF { break; case 35: - const imageTag = await readImagesJPEG(35, tag); + const jpegTag = await readImagesJPEG(35, tag); this._tags.push(new ImageTag({ - code: imageTag.code, - characterID: imageTag.characterId, - imgType: imageTag.imgType, - imgData: imageTag.imgData + code: jpegTag.code, + characterID: jpegTag.characterId, + imgType: jpegTag.imgType, + imgData: jpegTag.imgData })); break; case 36: - const imageTag: any = await readImagesDefineBitsLossless(tag); + const pngTag: any = await readImagesDefineBitsLossless(tag); this._tags.push(new ImageTag({ - code: imageTag.code, - characterID: imageTag.characterId, - imgType: imageTag.imgType, - imgData: imageTag.imgData + code: pngTag.code, + characterID: pngTag.characterId, + imgType: pngTag.imgType, + imgData: pngTag.imgData })); break; diff --git a/src/utils/SwfReader.ts b/src/utils/SwfReader.ts index bf61a45..64a5b28 100644 --- a/src/utils/SwfReader.ts +++ b/src/utils/SwfReader.ts @@ -1,23 +1,71 @@ const SWFReader = require('@gizeta/swf-reader'); -const {extractImage, test} = require("swf-extract"); -var _encoder = require('png-stream/encoder'); +const _encoder = require('png-stream/encoder'); -var _encoder2 = _interopRequireDefault(_encoder); +const _encoder2 = _interopRequireDefault(_encoder); -var _zlib = require('zlib'); +const _zlib = require('zlib'); -var _zlib2 = _interopRequireDefault(_zlib); +const _zlib2 = _interopRequireDefault(_zlib); -var _streamToArray = require('stream-to-array'); +const _streamToArray = require('stream-to-array'); -var _streamToArray2 = _interopRequireDefault(_streamToArray); +const _streamToArray2 = _interopRequireDefault(_streamToArray); -function _interopRequireDefault(obj: any) { return obj && obj.__esModule ? obj : { default: obj }; } +const _stream = require('stream'); -export function readSwfAsync(path: string): Promise { +const _stream2 = _interopRequireDefault(_stream); + +const _decoder = require('jpg-stream/decoder'); + +const _decoder2 = _interopRequireDefault(_decoder); + +function _interopRequireDefault(obj: any) { + return obj && obj.__esModule ? obj : {default: obj}; +} + +const _concatFrames = require('concat-frames'); + +const _concatFrames2 = _interopRequireDefault(_concatFrames); + +const _slicedToArray = function () { + function sliceIterator(arr: any, i: any) { + var _arr = []; + var _n = true; + var _d = false; + var _e = undefined; + try { + for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { + _arr.push(_s.value); + if (i && _arr.length === i) break; + } + } catch (err) { + _d = true; + _e = err; + } finally { + try { + if (!_n && _i["return"]) _i["return"](); + } finally { + if (_d) throw _e; + } + } + return _arr; + } + + return function (arr: any, i: any) { + if (Array.isArray(arr)) { + return arr; + } else if (Symbol.iterator in Object(arr)) { + return sliceIterator(arr, i); + } else { + throw new TypeError("Invalid attempt to destructure non-iterable instance"); + } + }; +}(); + +export function readSwfAsync(data: string | Buffer): Promise { return new Promise(((resolve, reject) => { - SWFReader.read(path, function (err: Error, swf: any) { + SWFReader.read(data, function (err: Error, swf: any) { if (err) { reject(err); } @@ -26,9 +74,94 @@ export function readSwfAsync(path: string): Promise { })); } +const pngMagic = Buffer.from('0x89 0x50 0x4E 0x47 0x0D 0x0A 0x1A 0x0A'.split(' ').map(Number)); +const gifMagic = Buffer.from('0x47 0x49 0x46 0x38 0x39 0x61'.split(' ').map(Number)); +const recognizeHeader = function recognizeHeader(buffer: Buffer) { + if (pngMagic.equals(buffer.slice(0, pngMagic.length))) return 'png'; + if (gifMagic.equals(buffer.slice(0, gifMagic.length))) return 'gif'; + return 'jpeg'; +}; -export async function readImagesJPEG(code: number, tag: any) { - return test(code)(tag); +export async function readImagesJPEG(code: number, tagData: any): Promise { + var characterId = tagData.characterId, + imageData = tagData.imageData; + + var imgType = recognizeHeader(imageData); + if (imgType !== 'jpeg') { + return { + code: code, + characterId: characterId, + imgType: imgType, + imgData: imageData + }; + } + + var bitmapAlphaData = tagData.bitmapAlphaData; + + return new Promise(function (resolve, reject) { + var enc = new _encoder2.default(undefined, undefined, {colorSpace: 'rgba'}); + _zlib2.default.unzip(bitmapAlphaData, function (err: any, alphaBufPre: any) { + // INVARIANT: alphaBuf is either null or a non-empty buffer + let alphaBuf: any = null; + if (err) { + /* + Due to a bug present in node zlib (https://github.com/nodejs/node/issues/17041) + unzipping an empty buffer can raise "unexpected end of file" error. + We fix this here so that our impl does not depend on the version of node + being used. + Theoretically every zlib.unzip call needs to be guarded, but for this package, + other two zlib.unzip call happens at sites that an empty uncompressed Buffer + does not make sense. So I think the current fix is good enough. + */ + if (bitmapAlphaData.length > 0) { + return reject(new Error(err)); + } + // leaving alphaBuf as null + } else { + // ensure alphaBuf is only assigned an non-empty Buffer + if (alphaBufPre.length > 0) alphaBuf = alphaBufPre; + } + var bufferStream = new _stream2.default.PassThrough(); + bufferStream.end(imageData); + bufferStream.pipe(new _decoder2.default()).pipe((_concatFrames2.default)(function (_ref: any) { + var _ref2 = _slicedToArray(_ref, 1), + frame = _ref2[0]; + + var input = frame.pixels; + var pCount = frame.width * frame.height; + var output = Buffer.alloc(pCount * 4); + if (alphaBuf !== null && alphaBuf.length !== pCount) { + console.error('expect alphaBuf to have size ' + pCount + ' while getting ' + alphaBuf.length); + } + var getAlphaBuf = alphaBuf === null ? function (_ignored: any) { + return 0xff; + } : function (i: any) { + return alphaBuf[i]; + }; + + for (var i = 0; i < pCount; ++i) { + output[4 * i] = input[3 * i]; + output[4 * i + 1] = input[3 * i + 1]; + output[4 * i + 2] = input[3 * i + 2]; + output[4 * i + 3] = getAlphaBuf(i); + } + enc.format.width = frame.width; + enc.format.height = frame.height; + enc.end(output); + })); + }); + (_streamToArray2.default)(enc).then(function (parts: any) { + var buffers = parts.map(function (part: any) { + return Buffer.isBuffer(part) ? part : Buffer.from(part); + }); + resolve({ + code: code, + characterId: characterId, + imgType: 'png', + imgData: Buffer.concat(buffers) + }); + }); + }); } export function readImagesDefineBitsLossless(tag: any) { @@ -47,7 +180,7 @@ export function readImagesDefineBitsLossless(tag: any) { if (err) { return reject(new Error(err)); } - var output = new Buffer(bitmapWidth * bitmapHeight * 4); + var output = Buffer.alloc(bitmapWidth * bitmapHeight * 4); var index = 0; var ptr = 0; if (bitmapFormat === 5) {