Download households, lots and rooms from The Sims 4 Gallery website
// ==UserScript== // @name TS4 gallery downloader // @description Download households, lots and rooms from The Sims 4 Gallery website // @author anadius // @match *://www.ea.com/*/games/the-sims/the-sims-4/pc/gallery* // @match *://www.ea.com/games/the-sims/the-sims-4/pc/gallery* // @connect sims4cdn.ea.com // @connect athena.thesims.com // @connect www.thesims.com // @connect thesims-api.ea.com // @version 2.1.12 // @namespace anadius.github.io // @grant unsafeWindow // @grant GM.xmlHttpRequest // @grant GM_xmlhttpRequest // @grant GM.getResourceUrl // @grant GM_getResourceURL // @icon https://anadius.github.io/ts4installer-tumblr-files/userjs/sims-4-gallery-downloader.png // @resource bundle.json https://anadius.github.io/ts4installer-tumblr-files/userjs/bundle.min.json?version=1.105.332 // @require https://greasemonkey.github.io/gm4-polyfill/gm4-polyfill.js // @require https://cdn.jsdelivr.net/npm/[email protected]/dist/long.js#sha256-Cp9yM71yBwlF4CLQBfDKHoxvI4BoZgQK5aKPAqiupEQ= // @require https://cdn.jsdelivr.net/npm/[email protected]/dist/FileSaver.min.js#sha256-Sf4Tr1mzejErqH+d3jzEfBiRJAVygvjfwUbgYn92yOU= // @require https://cdn.jsdelivr.net/npm/[email protected]/dist/jszip.min.js#sha256-VwkT6wiZwXUbi2b4BOR1i5hw43XMzVsP88kpesvRYfU= // @require https://cdn.jsdelivr.net/npm/[email protected]/dist/protobuf.min.js#sha256-VPK6lQo4BEjkmYz6rFWbuntzvMJmX45mSiLXgcLHCLE= // ==/UserScript== /* global protobuf, saveAs, JSZip, Long */ /* eslint curly: 0 */ /* eslint no-sequences: 0 */ /* eslint no-return-assign: 0 */ const TRAY_ITEM_URL = 'https://thesims-api.ea.com/api/gallery/v1/sims/{UUID}'; const DATA_ITEM_URL = 'http://sims4cdn.ea.com/content-prod.ts4/prod/{FOLDER}/{GUID}.dat'; const IMAGE_URL = 'https://athena.thesims.com/v2/images/{TYPE}/{FOLDER}/{GUID}/{INDEX}.jpg'; const EXCHANGE_HOUSEHOLD = 1; const EXCHANGE_BLUEPRINT = 2; const EXCHANGE_ROOM = 3; const EXTENSIONS = { [EXCHANGE_HOUSEHOLD]: ['Household', 'householdbinary', 'hhi', 'sgi'], [EXCHANGE_BLUEPRINT]: ['Lot', 'blueprint', 'bpi', 'bpi'], [EXCHANGE_ROOM]: ['Room', 'room', 'rmi', null] }; const BIG_WIDTH = 591; const BIG_HEIGHT = 394; const SMALL_WIDTH = 300; const SMALL_HEIGHT = 200; const LONG_TYPES = [ "sint64", "uint64", "int64", "sfixed64", "fixed64" ]; /* helper functions */ const getRandomIntInclusive = (min, max) => { min = Math.ceil(min); max = Math.floor(max); return Math.floor(Math.random() * (max - min + 1)) + min; }; const reportError = e => { if(e.name && e.message && e.stack) alert(`${e.name}\n\n${e.message}\n\n${e.stack}`); else alert(e); }; const xhr = details => new Promise((resolve, reject) => { const stack = new Error().stack; const reject_xhr = res => { console.log(res); reject({ name: 'GMXHRError', message: `XHR for URL ${details.url} returned status code ${res.status}`, stack: stack, status: res.status }); }; GM.xmlHttpRequest(Object.assign( {method: 'GET'}, details, { onload: res => { if(res.status === 404) reject_xhr(res); else resolve(res.response); }, onerror: res => reject_xhr } )); }); /* functions taken from thesims.min.js */ function dashify(uuid) { var slice = String.prototype.slice, indices = [ [0, 8], [8, 12], [12, 16], [16, 20], [20] ]; return indices.map(function(index) { return slice.apply(uuid, index) }).join("-") } function uuid2Guid(uuid) { if (-1 !== uuid.indexOf("-")) return uuid.toUpperCase(); var decoded; try { decoded = atob(uuid) } catch (err) { return !1 } for (var guid = "", i = 0; i < decoded.length; i++) { var ch = decoded.charCodeAt(i); ch = (240 & ch) >> 4, ch = ch.toString(16); var tmpstr = ch.toString(); ch = decoded.charCodeAt(i), ch = 15 & ch, ch = ch.toString(16), tmpstr += ch.toString(), guid += tmpstr } return dashify(guid).toUpperCase() } function getFilePath(guid) { var bfnvInit = 2166136261; for (var fnvInit = bfnvInit, i = 0; i < guid.length; ++i) fnvInit += (fnvInit << 1) + (fnvInit << 4) + (fnvInit << 7) + (fnvInit << 8) + (fnvInit << 24), fnvInit ^= guid.charCodeAt(i); var result = (fnvInit >>> 0) % 1e4; return result = result.toString(16), result = "0x" + "00000000".substr(0, 8 - result.length) + result }; // wrap everything in async anonymous function (async () => { /* tray item */ const getRandomId = () => { return new Long( getRandomIntInclusive(1, 0xffffffff), getRandomIntInclusive(0, 0xffffffff), true); }; const createPrefix = num => { const arr = new ArrayBuffer(8); const view = new DataView(arr); view.setUint32(4, num, true); return new Uint8Array(arr); }; const parseValue = (value, fieldType, isParentArray) => { let _; let parsedValue = value; const valueType = typeof value; if(valueType === "object") { if(Array.isArray(value)) { if(isParentArray) { throw "No clue how to handle array of arrays" } parsedValue = parseMessageArray(value, fieldType); } else { [parsedValue, _] = parseMessageObj(value, fieldType); } } else if(valueType === "string") { if(fieldType === "string" || fieldType === "bytes") { // no processing needed } else if(LONG_TYPES.includes(fieldType)) { parsedValue = Long.fromValue(value); } else { // value is enum, lookup the enum and extract the actual value parsedValue = root.lookupEnum(fieldType).values[value.split('.').pop()]; } } return parsedValue; }; const parseMessageArray = (messageArray, className) => { const parsedArray = []; messageArray.forEach(arrayItem => { parsedArray.push(parseValue(arrayItem, className, true)); }); return parsedArray; }; const camelToSnakeCase = str => str.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`); // EA likes to mess things up, in proto files they use snake case, in JSON they use camel case // but there are some exceptions, so try both const findKeyName = (key, messageClass) => { const fields = messageClass.fields; if(typeof messageClass.fields[key] !== "undefined") return key; const key2 = camelToSnakeCase(key); if(typeof messageClass.fields[key2] !== "undefined") return key2; const nameParts = []; let msgClass = messageClass; while(msgClass.name !== "" && typeof msgClass.name !== "undefined") { nameParts.unshift(msgClass.name); msgClass = msgClass.parent; } const name = nameParts.join("."); throw `${name} class doesn't have ${key} nor ${key2} key.` }; const parseMessageObj = (messageObj, className) => { const keys = Object.keys(messageObj); const messageClass = root.lookupType(className); const parsedMessage = {}; for(let i=0, l=keys.length; i<l; ++i) { // this makes sure that `key` is in `messageClass.fields` const key = findKeyName(keys[i], messageClass); parsedMessage[key] = parseValue(messageObj[keys[i]], messageClass.fields[key].type); } return [parsedMessage, messageClass]; }; const getTrayItem = async (uuid, guid, folder) => { let message; try { message = await xhr({ url: TRAY_ITEM_URL.replace('{UUID}', encodeURIComponent(uuid)), responseType: 'json', headers: { 'Accept-Language': 'en-US,en;q=0.9', 'Cookie': '' } }); } catch(e) { if(e.name === 'GMXHRError') message = null; else throw e; } if(message === null || typeof message.error !== 'undefined') throw "Can't download tray file. This item was most probably deleted."; const [parsedMessage, messageClass] = parseMessageObj(message, "EA.Sims4.Network.TrayMetadata"); // generate random ID parsedMessage.id = getRandomId(); // get number of additional images; set sim IDs let additional = 0; if(parsedMessage.type === EXCHANGE_BLUEPRINT) additional = parsedMessage.metadata.bp_metadata.num_thumbnails - 1; else if(parsedMessage.type === EXCHANGE_HOUSEHOLD) { additional = parsedMessage.metadata.hh_metadata.sim_data.length; // the same logic of making new IDs is used later to fix the main data file parsedMessage.metadata.hh_metadata.sim_data.forEach((sim, i) => { sim.id = parsedMessage.id.add(i + 1); }); } const encodedMessage = messageClass.encode(parsedMessage).finish(); const prefix = createPrefix(encodedMessage.byteLength); const resultFile = new Uint8Array(prefix.length + encodedMessage.length); resultFile.set(prefix); resultFile.set(encodedMessage, prefix.length); return [ resultFile, parsedMessage.type, parsedMessage.id, additional, parsedMessage.modifier_name || parsedMessage.creator_name, parsedMessage.name ]; }; /* data file */ const getDataItem = async (guid, folder, type, id) => { let response; try { response = await xhr({ url: DATA_ITEM_URL.replace('{FOLDER}', folder).replace('{GUID}', guid), responseType: 'arraybuffer' }); } catch(e) { if(e.name === 'GMXHRError' && e.status === 404) throw "Can't download data file. This item was most probably deleted."; else throw e; } if(type === EXCHANGE_HOUSEHOLD) { const messageClass = root.lookupTypeOrEnum('EA.Sims4.Network.FamilyData'); const prefix = new Uint8Array(response, 0, 4); // read first 4 bytes const view = new DataView(response); let len = view.getUint32(4, true); // from next 4 bytes read length const message = messageClass.decode(new Uint8Array(response, 8, len)); // read and decode message const suffix = new Uint8Array(response, 8 + len); // read the rest const newIdsDict = {}; const sims = message.family_account.sim; sims.forEach((sim, i) => { newIdsDict[sim.sim_id.toString()] = id.add(1+i); }); sims.forEach(sim => { sim.sim_id = newIdsDict[sim.sim_id.toString()]; sim.significant_other = newIdsDict[sim.significant_other.toString()]; sim.attributes.genealogy_tracker.family_relations.forEach(relation => { const newId = newIdsDict[relation.sim_id.toString()]; if(typeof newId !== "undefined") { relation.sim_id = newId; } }); }); try { const editedMessage = new Uint8Array(messageClass.encode(message).finish()); const resultArray = new Uint8Array(8 + editedMessage.length + suffix.length); resultArray.set(prefix); (new DataView(resultArray.buffer)).setUint32(4, editedMessage.length, true); resultArray.set(editedMessage, 8); resultArray.set(suffix, 8 + editedMessage.length); return resultArray.buffer; } catch(err) { console.error(err); return response; } } else return response; }; /* image files */ const loadImage = url => new Promise(resolve => { xhr({ url: url, responseType: 'blob' }).then(response => { const urlCreator = window.URL || window.webkitURL; const imageUrl = urlCreator.createObjectURL(response); const img = new Image(); img.onload = () => { urlCreator.revokeObjectURL(img.src); resolve(img); }; img.src = imageUrl; }); }); const newCanvas = (width, height) => { const canvas = document.createElement('canvas'); canvas.width = width; canvas.height = height; return canvas; }; const getImages = async (guid, folder, type, additional) => { const URL_TEMPLATE = IMAGE_URL.replace('{FOLDER}', folder).replace('{GUID}', guid).replace('{TYPE}', type - 1); const big = newCanvas(BIG_WIDTH, BIG_HEIGHT); const small = newCanvas(SMALL_WIDTH, SMALL_HEIGHT); const images = []; for(let i=0; i<=additional; ++i) { let url = URL_TEMPLATE.replace('{INDEX}', i.toString().padStart(2, '0')); let img = await loadImage(url); let x, y, width, height; if(type == EXCHANGE_BLUEPRINT || (type == EXCHANGE_HOUSEHOLD && i > 0)) { width = Math.round(img.naturalHeight * BIG_WIDTH / BIG_HEIGHT); height = img.naturalHeight; } else { width = BIG_WIDTH; height = BIG_HEIGHT; } x = (img.naturalWidth - width) / 2; y = (img.naturalHeight - height) / 2; if(i == 0) { small.getContext('2d').drawImage(img, x, y, width, height, 0, 0, SMALL_WIDTH, SMALL_HEIGHT); images.push(small.toDataURL('image/jpeg').split('base64,')[1]); } big.getContext('2d').drawImage(img, x, y, width, height, 0, 0, BIG_WIDTH, BIG_HEIGHT); images.push(big.toDataURL('image/jpeg').split('base64,')[1]); } return images; }; /* main download */ const generateName = (type, id, ext) => { const typeStr = '0x' + type.toString(16).toLowerCase().padStart(8, 0); const idStr = '0x' + id.toString(16).toLowerCase().padStart(16, 0); return typeStr + '!' + idStr + '.' + ext; }; const toggleDownload = (scope, downloading) => { scope.vm.toggleDownload.toggling = downloading; scope.$apply(); }; const downloadItem = async scope => { try { const uuid = scope.vm.uuid; const guid = uuid2Guid(uuid); const folder = getFilePath(guid); toggleDownload(scope, true); const zip = new JSZip(); const [trayItem, type, id, additional, author, title] = await getTrayItem(uuid, guid, folder); zip.file(generateName(type, id, 'trayitem'), trayItem); const [typeStr, dataExt, imageExt, additionalExt] = EXTENSIONS[type]; const dataItem = await getDataItem(guid, folder, type, id); zip.file(generateName(0, id, dataExt), dataItem); const images = await getImages(guid, folder, type, additional); images.forEach((data, i) => { let group = i == 0 ? 2 : 3; let extension = i < 2 ? imageExt : additionalExt; let newId = id; if(i >= 2) { let j = i - 1; group += (1 << (4 * type)) * j; if(type == EXCHANGE_HOUSEHOLD) newId = newId.add(j); } zip.file(generateName(group, newId, extension), data, {base64: true}); }); let filename = [author, typeStr, title, uuid.replace(/\+/g, '-').replace(/\//g, '_')].join('__'); filename = filename.replace(/\s+/g, '_').replace(/[^a-z0-9\.\-=_]/gi, ''); const content = await zip.generateAsync({type:'blob'}); saveAs(content, filename + '.zip'); } catch(e) { reportError(e); } toggleDownload(scope, false); }; /* init */ let data = await fetch(await GM.getResourceUrl('bundle.json')); let jsonDescriptor = await data.json(); const root = protobuf.Root.fromJSON(jsonDescriptor); document.addEventListener('click', e => { let el = e.target; if(el.tagName === 'SPAN') el = el.parentNode.parentNode; else if(el.tagName === 'A') el = el.parentNode; if(el.tagName === 'LI' && el.classList.contains('stream-tile__actions-download')) { e.stopPropagation(); const scope = unsafeWindow.angular.element(el).scope(); downloadItem(scope); } }, true); console.log('running'); })(); /* add "force login" link */ const a = document.createElement('a'); a.href = 'https://www.thesims.com/login?redirectUri=' + encodeURIComponent(document.location); a.innerHTML = '<b>force login</b>'; a.style.background = 'grey'; a.style.color = 'white'; a.style.display = 'inline-block'; a.style.position = 'absolute'; a.style.top = 0; a.style.left = 0; a.style.height = '40px'; a.style.lineHeight = '40px'; a.style.padding = '0 15px'; a.style.zIndex = 99999; document.body.appendChild(a);