try {

const spp = new SentencePieceProcessor();

await spp.load(modelPath);

return spp;

} catch (error) {

console.error("Sentencepiece tokenizer failed to load: " + modelPath, error);

return null;

}

// Fallback to strlen estimation

if (!spp) {

return Math.ceil(text.length / CHARS_PER_TOKEN);

}

let cleaned = cleanText(text);

let ids = spp.encodeIds(cleaned);

return ids.length;

try {

const arrayBuffer = fs.readFileSync(modelPath).buffer;

const instance = await Tokenizer.fromJSON(arrayBuffer);

return instance;

} catch (error) {

console.error("Claude tokenizer failed to load: " + modelPath, error);

return null;

}

const convertedPrompt = convertClaudePrompt(messages, false, false);

// Fallback to strlen estimation

if (!tokenizer) {

return Math.ceil(convertedPrompt.length / CHARS_PER_TOKEN);

}

const count = tokenizer.encode(convertedPrompt).length;

return count;

if (requestModel.includes('claude')) {

return 'claude';

}

if (requestModel.includes('gpt-4-32k')) {

return 'gpt-4-32k';

}

if (requestModel.includes('gpt-4')) {

return 'gpt-4';

}

if (requestModel.includes('gpt-3.5-turbo')) {

return 'gpt-3.5-turbo';

}

// default

return 'gpt-3.5-turbo';

if (tokenizersCache[model]) {

return tokenizersCache[model];

}

const tokenizer = tiktoken.encoding_for_model(model);

console.log('Instantiated the tokenizer for', model);

tokenizersCache[model] = tokenizer;

return tokenizer;

let baseDate = new Date(Date.now());

let humanYear = baseDate.getFullYear();

let humanMonth = (baseDate.getMonth() + 1);

let humanDate = baseDate.getDate();

let humanHour = (baseDate.getHours() < 10 ? '0' : '') + baseDate.getHours();

let humanMinute = (baseDate.getMinutes() < 10 ? '0' : '') + baseDate.getMinutes();

let humanSecond = (baseDate.getSeconds() < 10 ? '0' : '') + baseDate.getSeconds();

let humanMillisecond = (baseDate.getMilliseconds() < 10 ? '0' : '') + baseDate.getMilliseconds();

let HumanizedDateTime = (humanYear + "-" + humanMonth + "-" + humanDate + " @" + humanHour + "h " + humanMinute + "m " + humanSecond + "s " + humanMillisecond + "ms");

return HumanizedDateTime;

const streamingUrl = request.header('X-Streaming-URL');

const websocket = new WebSocket(streamingUrl);

websocket.on('open', async function () {

console.log('websocket open');

websocket.send(JSON.stringify(request.body));

});

websocket.on('error', (err) => {

console.error(err);

websocket.close();

});

websocket.on('close', (code, buffer) => {

const reason = new TextDecoder().decode(buffer)

console.log(reason);

});

while (true) {

if (isGenerationStopped) {

console.error('Streaming stopped by user. Closing websocket...');

websocket.close();

return;

}

const rawMessage = await new Promise(resolve => websocket.once('message', resolve));

const message = json5.parse(rawMessage);

switch (message.event) {

case 'text_stream':

yield message.text;

break;

case 'stream_end':

websocket.close();

return;

}

}

let pkgVersion = 'UNKNOWN';

let gitRevision = null;

let gitBranch = null;

try {

const pkgJson = require('./package.json');

pkgVersion = pkgJson.version;

if (!process.pkg && commandExistsSync('git')) {

gitRevision = require('child_process')

.execSync('git rev-parse --short HEAD', { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'ignore'] })

.toString().trim();

gitBranch = require('child_process')

.execSync('git rev-parse --abbrev-ref HEAD', { cwd: process.cwd(), stdio: ['ignore', 'pipe', 'ignore'] })

.toString().trim();

}

}

catch {

// suppress exception

}

const agent = `SillyTavern:${pkgVersion}:Cohee#1207`;

return { agent, pkgVersion, gitRevision, gitBranch };

try {

return json5.parse(str);

} catch {

return undefined;

}

// Simulate incoming data from frontend form

const result = charaFormatData({

json_data: JSON.stringify(char),

ch_name: char.name,

description: char.description,

personality: char.personality,

scenario: char.scenario,

first_mes: char.first_mes,

mes_example: char.mes_example,

creator_notes: char.creatorcomment,

talkativeness: char.talkativeness,

fav: char.fav,