lib/backtester.js

let _ = require('lodash')
  , moment = require('moment')
  , argv = require('yargs').argv
  , tb = require('timebucket')
  , readline = require('readline')
  , z = require('zero-fill')
  , n = require('numbro')
  , shell = require('shelljs')
  , StripAnsi = require('strip-ansi')
  , path = require('path')
  , fs = require('fs')
  , roundp = require('round-precision')
  , Phenotypes = require('./phenotype')


const spawn = require('child_process').spawn

let simArgs, simTotalCount, parallelLimit, writeFile

let processOutput = function (output, taskStrategyName, pheno) {
  let selector = pheno.selector || pheno.exchangeMarketPair
  let tFileName = path.resolve(__dirname, '..', 'simulations', 'sim_' + taskStrategyName.replace('_', '') + '_' + selector.toLowerCase().replace('_', '') + '_' + pheno.backtester_generation + '.json')
  let simulationResults

  let outputArray
  let params
  let assetPriceLastBuy
  let lastAssestValue
  let assetCapital
  let profit
  let startCapital
  let endBalance
  let buyHold
  let vsBuyHold
  //let wlMatch
  //let errMatch
  let wins
  let losses
  let errorRate
  let days
  let start
  let end
  // This can retrieve the results from 2 different places.  It defaults to reading it from the json file
  // but if no file is found it will fall back to the older metheod of scraping the output of the sim process
  // stdio scraping to be removed after full verification of functionality.
  // todo: see above comment
  if (fs.existsSync(tFileName)) {
    let jsonBuffer
    jsonBuffer = fs.readFileSync(tFileName, { encoding: 'utf8' })
    simulationResults = JSON.parse(jsonBuffer)
    fs.unlinkSync(tFileName)
  }

  // If somehow the sim file failed to write, this will most often recover it by parsing the last output
  if (typeof (simulationResults) !== 'object') {
    // Find everything between the first { and last }
    outputArray = output.split('{')
    outputArray.shift()
    output = outputArray.join('{')

    outputArray = output.split('}')
    outputArray.pop()
    output = outputArray.join('}')

    simulationResults = JSON.parse(`{${output}}`)
  }

  if (typeof (simulationResults) === 'object' && typeof simulationResults.simresults !== typeof undefined) {
    params = simulationResults
    endBalance = simulationResults.simresults.currency
    assetPriceLastBuy = simulationResults.simresults.last_buy_price
    lastAssestValue = simulationResults.simresults.last_assest_value
    assetCapital = simulationResults.simresults.asset_capital
    startCapital = simulationResults.simresults.start_capital
    profit = simulationResults.simresults.profit


    buyHold = simulationResults.simresults.buy_hold
    vsBuyHold = simulationResults.simresults.vs_buy_hold
    //wlMatch = (simulationResults.simresults.total_sells - simulationResults.simresults.total_losses) +'/'+ simulationResults.simresults.total_losses
    wins = simulationResults.simresults.total_sells - simulationResults.simresults.total_losses
    losses = simulationResults.simresults.total_losses
    errorRate = simulationResults.simresults.total_losses / simulationResults.simresults.total_sells
    days = parseInt(simulationResults.days)
    start = parseInt(simulationResults.start)
    end = parseInt(simulationResults.end || null)
  }
  else {
    console.log(`Couldn't find simulationResults for ${pheno.backtester_generation}`)
    console.log(pheno.command.commandString)
    // this should return a general bad result but not throw an error
    // our job here is to use the result.  not diagnose an error at this point so a failing sim should just be ignored.
    // idea here is to make the fitness of this calculation as bad as possible so darwin won't use the combonation of parameters again.
    // todo:  make the result its own object, and in this function just set the values don't define the result here.
    return {
      params: 'module.exports = {}',
      endBalance: 0,
      buyHold: 0,
      vsBuyHold: 0,
      lastAssestValue: 0,
      assetPriceLastBuy:0,
      wins: 0,
      losses: -1,
      errorRate: 100,
      days: 0,
      period_length: 0,
      min_periods: 0,
      markdown_buy_pct: 0,
      markup_sell_pct: 0,
      order_type: 'maker',
      wlRatio: 'Infinity',
      roi: -1000,
      selector: selector,
      strategy: taskStrategyName,
      frequency: 0,
      assetCapital:0,
      startCapital:0,
      profit:0

    }
  }

  if (typeof params === 'undefined') {
    console.log('busted params')
    console.log(`output: ${JSON.stringify(output)}`)
    console.log(`simulationResults: ${JSON.stringify(simulationResults)}`)
  }

  let roi
  if (params.currency_capital == 0.0) {
    roi = roundp(endBalance, 3)
  }
  else {
    roi = roundp(((endBalance - params.currency_capital) / params.currency_capital) * 100, 3)
  }

  //todo: figure out what this is trying to do.
  let r = params
  delete r.asset_capital
  delete r.buy_pct
  delete r.currency_capital
  delete r.days
  delete r.mode
  delete r.order_adjust_time
  delete r.population
  delete r.population_data
  delete r.sell_pct
  delete r.start
  delete r.end
  delete r.stats
  delete r.use_strategies
  delete r.verbose
  delete r.simresults
  delete r.silent
  delete r.generateLaunch
  delete r.ignoreLaunchFitness
  delete r.maxCores
  delete r.minTrades
  delete r.noStatSave
  delete r.filename
  //delete r.fitnessCalcType
   
  r.selector = r.selector.normalized

  if (start) {
    r.start = moment(start).format('YYYYMMDDHHmm')
  }
  if (end) {
    r.end = moment(end).format('YYYYMMDDHHmm')
  }
  if (!start && !end && params.days) {
    r.days = params.days
  }
  if (!days) {
    days = parseInt(argv.days, 10)
  }
  if (!days || days < 1) days = 1

  let results = {
    params: 'module.exports = ' + JSON.stringify(r),
    assetPriceLastBuy: assetPriceLastBuy,
    lastAssestValue: lastAssestValue,
    profit: profit,
    assetCapital: assetCapital,
    startCapital: startCapital,
    endBalance: parseFloat(endBalance),
    buyHold: parseFloat(buyHold),
    vsBuyHold: parseFloat(vsBuyHold) || vsBuyHold,
    wins: wins,
    losses: losses,
    errorRate: parseFloat(errorRate),
    days: days,
    period_length: params.period_length,
    min_periods: params.min_periods,
    markdown_buy_pct: params.markdown_buy_pct,
    markup_sell_pct: params.markup_sell_pct,
    order_type: params.order_type,
    wlRatio: losses > 0 ? roundp(wins / losses, 3) : 'Infinity',
    roi: roi,
    selector: params.selector,
    strategy: params.strategy,
    frequency: roundp((wins + losses) / days, 3)
  }


  return results
}

let runUpdate = function (days, selector) {
  let zenbot_cmd = process.platform === 'win32' ? 'zenbot.bat' : './zenbot.sh'
  let command = `${zenbot_cmd} backfill --days=${days} ${selector}`
  console.log('Backfilling (might take some time) ...')
  console.log(command)

  shell.exec(command, {
    silent: true,
    async: false
  })
}

let ensureDirectoryExistence = function (filePath) {
  var dirname = path.dirname(filePath)
  if (fs.existsSync(dirname)) {
    return true
  }
  ensureDirectoryExistence(dirname)
  fs.mkdirSync(dirname)
}

let monitor = {
  periodDurations: [],
  phenotypes: [],

  distanceOfTimeInWords: (timeA, timeB) => {
    var hourDiff = timeA.diff(timeB, 'hours')
    let minDiff = 0
    if (hourDiff == 0) {
      minDiff = timeA.diff(timeB, 'minutes')
      var secDiff = timeA.clone().subtract(minDiff, 'minutes').diff(timeB, 'seconds')
      return `${minDiff}m ${secDiff}s`
    }
    else {
      minDiff = timeA.clone().subtract(hourDiff, 'hours').diff(timeB, 'minutes')
      return `${hourDiff}h ${minDiff}m`
    }
  },

  actualRange: function (so) {
    // Adapted from sim.js logic to similarly figure out how much time is being processed
    if (so.start) {
      so.start = moment(so.start, 'YYYYMMDDHHmm')
      if (so.days && !so.end) {
        so.end = so.start.clone().add(so.days, 'days')
      }
    }
    if (so.end) {
      so.end = moment(so.end, 'YYYYMMDDHHmm')
      if (so.days && !so.start) {
        so.start = so.end.clone().subtract(so.days, 'days')
      }
    }
    if (!so.start && so.days) {
      so.start = moment().subtract(so.days, 'days')
    }

    if (so.days && !so.end) {
      so.end = so.start.clone().add(so.days, 'days')
    }

    if (so.start && so.end) {
      var actualStart = moment(tb(so.start.valueOf()).resize(so.period_length).subtract(so.min_periods + 2).toMilliseconds())
      return {
        start: actualStart,
        end: so.end
      }
    }

    return { start: so.start, end: so.end }
  },

  reportStatus: function () {
    var genCompleted = 0
    // var genTotal = 0

    var simsDone = 0
    var simsActive = 0
    var simsErrored = 0
    var simsAll = simTotalCount
    var simsRemaining = simsAll
    // var self = this
    // console.log(`simsAll: ${simsAll}, this.phenotypes: ${this.phenotypes.length}`);

    readline.clearLine(process.stdout)
    readline.cursorTo(process.stdout, 0)

    var inProgress = []
    var inProgressStr = []

    var slowestP = null
    var slowestEta = null

    var bestP = null
    var bestBalance = null

    this.phenotypes.forEach(function (p) {
      if ('sim' in p) {
        if (Object.keys(p.sim).length === 0) {
          simsActive++
          inProgress.push(p)
        }
        else {
          simsDone++

          if (!p.command || !p.command.result)
            simsErrored++

          if (p.command) {
            let balance = p.command.result.endBalance

            if (bestP == null || bestBalance < balance) {
              bestP = p
              bestBalance = balance
            }
            else if (bestP && bestBalance == balance && bestP.command.iteration > p.command.iteration) {
              // Always pick the earliest one so it doesn't look like the number is jumping all over the place
              bestP = p
              bestBalance = balance
            }
          }
        }
        simsRemaining--
      }

    })

    var homeStretchMode = simsActive < (parallelLimit - 1) && simsRemaining == 0

    inProgress.forEach(function (p) {
      var c = p.command

      var currentTime
      if (c.currentTimeString) currentTime = moment(c.currentTimeString, 'YYYY-MM-DD HH:mm:ss')
      if (currentTime && currentTime.isBefore(c.queryStart)) c.queryStart = currentTime
      // console.log(`${c.iteration} currentTime: ${currentTime}, queryStart: ${c.queryStart}, queryEnd: ${c.queryEnd}, current: ${c.currentTimeString}`);

      // var timeSoFar = moment().diff(c.startTime);
      // console.log(`remaining: ${time} - ${timeSoFar} = ${time - timeSoFar}`);
      // timeLeft += time - timeSoFar;
      if (currentTime && c.queryStart && c.queryEnd) {
        var totalTime = c.queryEnd.diff(c.queryStart)

        // 2018-01-25 06:18:00
        var progress = currentTime.diff(c.queryStart)

        // console.log(`totalTime: ${totalTime} vs progress: ${progress}`);
        var percentage = Math.min(progress / totalTime, 1)
        genCompleted += percentage

        var now = moment()
        var timeElapsed = now.diff(c.startTime)
        // console.log(`startTime: ${c.startTime}, timeElapsed: ${timeElapsed}, adding: ${timeElapsed / percentage}ms`);
        var eta = c.startTime.clone().add(timeElapsed / percentage, 'milliseconds')

        if (slowestP == null || slowestEta.isBefore(eta)) {
          slowestP = p
          slowestEta = eta
        }

        if (homeStretchMode)
          inProgressStr.push(`${(c.iteration + ':').gray} ${(percentage * 100).toFixed(1)}% ETA: ${monitor.distanceOfTimeInWords(eta, now)}`)
        else
          inProgressStr.push(`${(c.iteration + ':').gray} ${(percentage * 100).toFixed(1)}%`)
      }
    })


    // timeLeft /= simsActive; // how many run at one time
    if (inProgressStr.length > 0) {
      // process.stdout.write("\u001b[1000D") // Move left
      process.stdout.write('\u001b[1A')
      readline.clearLine(process.stdout)
      readline.cursorTo(process.stdout, 0)

      process.stdout.write(inProgressStr.join(', '))
      process.stdout.write('\n')
    }


    var percentage = ((simsDone + genCompleted) / simsAll * 100).toFixed(1)
    // z(8, n(s.period.trend_ema_rate).format('0.0000'), ' ')[color]
    process.stdout.write(`Done: ${simsDone.toString().green}, Active: ${simsActive.toString().yellow}, Remaining: ${simsRemaining.toString().gray}, `)
    if (simsErrored > 0)
      process.stdout.write(`Errored: ${simsErrored.toString().red}, `)

    process.stdout.write(`Completion: ${z(5, (n(percentage).format('0.0') + '%'), ' ').green} `)

    let bestBColor = 'gray'

    if (bestP) {
      if (argv.currency_capital) {
        let cc = parseFloat(argv.currency_capital)
        if (cc < 0.1)
          bestBColor = 'green'
        else if (cc > bestBalance)
          bestBColor = 'red'
        else
          bestBColor = 'yellow'
      }
    }

    let bestBalanceString = z(5, n(bestBalance || 0).format('0.0000'), ' ')[bestBColor]
    process.stdout.write(`Best Balance(${(bestP ? bestP.command.iteration.toString() : '?')[bestBColor]}): ${bestBalanceString}`)

    if (inProgressStr.length > 0) {
      if (!homeStretchMode)
        process.stdout.write(`, Slowest(${slowestP.command.iteration.toString().yellow}) ETA: ${monitor.distanceOfTimeInWords(slowestEta, moment()).yellow}`)
    }
  },

  reset: function () {
    this.phenotypes.length = 0
  },

  start: function () {
    process.stdout.write('\n\n')
    this.generationStarted = moment()

    this.reportInterval = setInterval(() => {
      this.reportStatus()
    }, 1000)
  },

  stop: function (label) {
    this.generationEnded = moment()
    clearInterval(this.reportInterval)
    var timeStr = this.distanceOfTimeInWords(this.generationEnded, this.generationStarted)
    console.log(`\n\n${label} completed at ${this.generationEnded.format('YYYY-MM-DD HH:mm:ss')}, took ${timeStr}, results saved to:`)
  }
}

module.exports = {

  init: function (options) {
    simArgs = options.simArgs
    simTotalCount = options.simTotalCount
    parallelLimit = options.parallelLimit
    writeFile = options.writeFile
  },

  deLint: function () {
    //Clean up any generation files left over in the simulation directory
    //they will be overwritten, but best not to confuse the issue.
    //if it fails.   doesn't matter they will be overwritten anyways. not need to halt the system.
    try {
      let tDirName = path.resolve(__dirname, '..', 'simulations')
      let tFileName = 'sim_'
      let files = fs.readdirSync(tDirName)

      for (let i = 0; i < files.length; i++) {
        if (files[i].lastIndexOf(tFileName) == 0) {
          let filePath = path.resolve(__dirname, '..', 'simulations', files[i])
          fs.unlinkSync(filePath)
        }

      }
    } catch (err) {
      console.log('error deleting lint from prior run')
    }
  },

  writeFileAndFolder: function (filePath, data) {
    ensureDirectoryExistence(filePath)
    fs.writeFileSync(filePath, data)
  },

  ensureBackfill: function () {
    let days = argv.days
    if (!days) {
      if (argv.start) {
        var start = moment(argv.start, 'YYYYMMDDHHmm')
        days = Math.max(1, moment().diff(start, 'days'))
      }
      else {
        var end = moment(argv.end, 'YYYYMMDDHHmm')
        days = moment().diff(end, 'days') + 1
      }
    }
    runUpdate(days, argv.selector)
  },

  buildCommand: function (taskStrategyName, phenotype, filename) {
    var cmdArgs = Object.assign({}, phenotype)
    cmdArgs.strategy = taskStrategyName
    Object.assign(cmdArgs, simArgs)

    var selector = cmdArgs.selector
    delete cmdArgs.selector
    delete cmdArgs.exchangeMarketPair
    delete cmdArgs.sim
    delete cmdArgs.command
    delete cmdArgs.help
    delete cmdArgs.version

    if (argv.include_html)
      cmdArgs.filename = filename

    if (argv.silent)
      cmdArgs.silent = true

    cmdArgs.backtester_generation = phenotype.backtester_generation

    let zenbot_cmd = process.platform === 'win32' ? 'zenbot.bat' : './zenbot.sh'
    let command = `${zenbot_cmd} sim ${selector}`

    for (const [key, value] of Object.entries(cmdArgs)) {
      if (_.isBoolean(value)) {
        command += ` --${value ? '' : 'no-'}${key}`
      } else {
        command += ` --${key}=${value}`
      }
    }

    var actualRange = monitor.actualRange({
      start: cmdArgs.start, end: cmdArgs.end, days: cmdArgs.days,
      period_length: cmdArgs.period_length, min_periods: (cmdArgs.min_periods || 1)
    })

    return {
      commandString: command,
      queryStart: actualRange.start,
      queryEnd: actualRange.end
    }
  },

  runCommand: (taskStrategyName, phenotype, command, cb) => {
    // console.log(`[ ${command.iteration}/${populationSize * selectedStrategies.length} ] ${command.commandString}`)

    phenotype['sim'] = {}
    phenotype['command'] = command

    command.startTime = moment()
    var cmdArgs = command.commandString.split(' ')
    var cmdName = cmdArgs.shift()
    const proc = spawn(cmdName, cmdArgs)
    var endData = ''

    proc.on('exit', () => {
      let result = null
      let stdout = endData.toString()
      try {
        result = processOutput(stdout, taskStrategyName, phenotype)

        command.endTime = moment()
        command.result = result

        writeFile(command.iteration, JSON.stringify(command))

        phenotype['sim'] = result
        result['fitness'] = Phenotypes.fitness(phenotype)

        monitor.reportStatus()

      } catch (err) {
        console.log(`Bad output detected on sim ${command.iteration} while running:`)
        console.log(command.commandString)
        console.log(err.toString())
        console.log(stdout)
        console.log(err.stack)
      }

      cb(null, result)
    })
    proc.stdout.on('data', (data) => {
      if (data.length > 500) {
        endData = data
        // console.log(`${command.iteration}: ${data}`)
      }
      else {
        var str = StripAnsi(data.toString()), lines = str.split(/(\r?\n)/g)
        for (var i = 0; i < lines.length; i++) {
          var line = lines[i]
          // console.log(`${command.iteration}: ${line}`)
          if (line.indexOf('-') == 4 && line.indexOf(':') == 13) {
            var timeStr = line.slice(0, 20)
            command.currentTimeString = timeStr
            // console.log(`${command.iteration}: ${command.currentTimeString}`)
          }
        }

      }
    })
  },

  startMonitor: () => monitor.start(),
  stopMonitor: (label) => monitor.stop(label),
  resetMonitor: () => monitor.reset(),
  reportStatus: () => monitor.reportStatus(),
  trackPhenotype: function (phenotype) {
    monitor.phenotypes.push(phenotype)
  }

}