civitai.info.js

/*  This script exists to merge all .info files into one array.
 *  unfortunately, civitai is storing some of them as LFS objects
 *  so the script outputs a list of errors along with scripts to fix them (delfiles.sh, replacefiles.sh)
 *  others files are blank, so just ignore the errors the second time, they're files that aren't on civitai
 *  now we get tot he important part: it parses all files, and dumps all of them into an array in civitai.info.json for further processing
 *  Run once, run delfiles.bat or delfiles.sh to get rid of the bad ones, then run replacefiles.sh to fetch them from civitai.
 */
var path = require('path'),
fs = require('fs');

function fromDir(startPath, filter, arr) {
    arr=arr||[];
    //console.log('Starting from dir '+startPath+'/');
    if (!fs.existsSync(startPath)) {
        console.log("no dir ", startPath);
        return;
    }

    var files = fs.readdirSync(startPath);
    for (var i = 0; i < files.length; i++) {
        var filename = path.join(startPath, files[i]);
        var stat = fs.lstatSync(filename);
        if (stat.isDirectory()) {
            fromDir(filename, filter,arr); //recurse
        } else if (filename.endsWith(filter)) {
            arr.push(filename);
            //console.log('-- found: ', filename);
        };
    };
    return arr;
};

var addons=[];
var errors=[];
var push=function(file){
	try{
		var addon=JSON.parse(fs.readFileSync(file));
		addon.mirror_path=file; // Add the location we found the file.
		addons.push(addon)
	} catch(err) {
		console.log(file);
		console.error(err)
		errors.push(file);
	}
};

var files=fromDir('.', '.civitai.info');
files.map(e=>{push(e);});

fs.writeFileSync('civitai.info.json',JSON.stringify(addons))
fs.writeFileSync('errors.json',JSON.stringify(errors))
//console.error("Error with the following files:",errors);
fs.writeFileSync('delfiles.bat','@echo off\n\n'+errors.map(e=>{return "del \""+ e + "\""}).join('\n'))
fs.writeFileSync('delfiles.sh',errors.map(e=>{return "rm \""+ e + "\""}).join('\n').split('\\').join('/'))
fs.writeFileSync('replacefiles.sh',errors.map(e=>{return "wget \"https://huggingface.co/anonderpling/civitai_mirror/resolve/main/"+(e.split(" ").join("%23")+"\" -O \""+e+"\"")}).join('\n').split('\\').join('\/'))