DMelt:Finance/Market Prediction

Table of contents

Market prediction

In Section Neural Network we discussed how to build a neural net based on backpropogation algorithm. Now we will show how to use this neural network approach to make predictions of financial market market.

Using Joone

Let us consider Joone Neural net for forecasting market predictions using a generated time series in the form cos(x)*sin(x)+Gaussian noise.

from org.joone.engine import NeuralNetListener,TanhLayer,DelayLayer 
from org.joone.engine import SigmoidLayer,FullSynapse,Monitor 
from org.joone.io import FileInputSynapse,FileOutputSynapse 
from org.joone.engine.learning import TeachingSynapse
from org.joone.net import NeuralNet
from java.lang import System
from java.io import *
from jhplot  import *
from java.awt import *
import math
from java.util import  Random


class joone(NeuralNetListener):

  mills=0
  epochs=1000
  fileName=""
  nnet = NeuralNet()
  input = DelayLayer()
  hidden = SigmoidLayer()
  output = TanhLayer()


  # a function to help connect layers
  def connect(self,layer1, syn, layer2):
        layer1.addOutputSynapse(syn)
        layer2.addInputSynapse(syn)

  def createDataSet(self,fileName,firstRow, lastRow, advColSel):
        fInput = FileInputSynapse()
        fInput.setInputFile(File(fileName))
        fInput.setFirstRow(firstRow)
        fInput.setLastRow(lastRow)
        fInput.setAdvancedColumnSelector(advColSel)
        return fInput


  def createNet(self,fileName,epochs,trainingPatterns,temporalWindow):
     self.epochs=epochs
     print "Running NN for ", epochs, " epochs"
     self.fileName=fileName
     self.input.setTaps(temporalWindow-1)
     self.input.setRows(1)
     self.hidden.setRows(15)
     self.output.setRows(1)

     self.connect(self.input,FullSynapse(), self.hidden)
     self.connect(self.hidden,FullSynapse(), self.output)

     self.input.addInputSynapse(self.createDataSet(self.fileName, 1, trainingPatterns, "1"))

     trainer = TeachingSynapse()
     trainer.setDesired(self.createDataSet(self.fileName, 2, trainingPatterns+1, "1"))
     self.output.addOutputSynapse(trainer)

     self.nnet.addLayer(self.input, NeuralNet.INPUT_LAYER)
     self.nnet.addLayer(self.hidden, NeuralNet.HIDDEN_LAYER)
     self.nnet.addLayer(self.output, NeuralNet.OUTPUT_LAYER)
     self.mills = System.currentTimeMillis()
     self.nnet.randomize(0.5) 


  def train(self): 
        mon = self.nnet.getMonitor()
        mon.setLearningRate(0.2)
        mon.setMomentum(0.7)
        mon.setTrainingPatterns(trainingPatterns)
        mon.setTotCicles(epochs)
        mon.setPreLearning(temporalWindow)
        mon.setLearning(True)
        mon.addNeuralNetListener(self)
        self.nnet.start()
        mon.Go()
        self.nnet.join()

  def interrogate(self,outputFile):
        mon = self.nnet.getMonitor()
        self.input.removeAllInputs()
        startRow = trainingPatterns - temporalWindow
        self.input.addInputSynapse(self.createDataSet(self.fileName, startRow+1, startRow+40, "1"))
        self.output.removeAllOutputs()
        fOutput = FileOutputSynapse()
        fOutput.setFileName(outputFile)
        self.output.addOutputSynapse(fOutput)
        mon.setTrainingPatterns(40)
        mon.setTotCicles(1)
        mon.setLearning(False)
        self.nnet.start()
        mon.Go()
        self.nnet.join()

  def netStopped(self,e):
      mon =e.getSource()
      if (mon.isLearning()): 
            epoch = mon.getTotCicles() - mon.getCurrentCicle()
            print "Epoch:",epoch," last RMSE=",mon.getGlobalError()
      else:
         delay = System.currentTimeMillis() - self.mills
         print "Training finished after ",delay," ms"

  def cicleTerminated(self,e):
      mon = e.getSource()
      epoch = mon.getTotCicles() - mon.getCurrentCicle()
      if ((epoch > 0) and ((epoch % 100) == 0)): 
            print "Epoch:",epoch," RMSE=",mon.getGlobalError()

  def netStarted(self,e):
      pass 
    
  def errorChanged(self,e):
      pass
 
  def netStoppedError(self,e):
      pass 

r=Random() 
print "Create a file joone_timeseries.txt with time series cos(x)*sin(x)+noise"
f = open("joone_timeseries.txt", "w")
for x in range(0,1000):
       yEst = 0.9*math.cos(x*0.4)*math.sin(0.7*x)+0.02*r.nextGaussian() # cos(x*0.3)*sim(x)+noise 
       f.write( str(yEst) + "\n"  )
f.close()


# get input file with TimeSeries 
fileName = "joone_timeseries.txt"
print "Downloading .. "+fileName
# Web.get("http://datamelt.org/examples/data/"+fileName)
trainingPatterns = 200
epochs = 10000
temporalWindow = 20

c1 = HPlot("show data")
c1.visible()
c1.setRangeX(0,1100)
c1.setNameX("time")
c1.setNameY("price")
c1.setMarginLeft(90)

# show data
dataIN = [line.strip() for line in open(fileName, 'r')]
print "Total data size=",len(dataIN), " For learning=",trainingPatterns
p1=P1D("Time series")
p1.setDrawLine(True)
p1.setPenWidth(1)
for i in range(len(dataIN)):
    p1.add(i,float(dataIN[i]))
c1.draw(p1)

# run NN for predictions
ts=joone()
ts.createNet(fileName,epochs,trainingPatterns,temporalWindow)
print "Training..."
ts.train()
ts.interrogate("results1.txt")
ts.interrogate("results2.txt")

# show predictions
data = [line.strip() for line in open("results2.txt", 'r')]
p2=P1D("Prediction")
p2.setColor(Color.red)
p2.setDrawLine(True)
for i in range(len(data)):
     p2.add(len(dataIN)+i,float(data[i]))

c1.draw(p2)
print "Done."

Here is the output image that shows the original time series and the predicted trends (n red).

As you can see, the neural net correctly predicts the expected behavior.

Market prediction

Now we consider another practical example. First, we will fetch data from Yahoo and store this data in a file: YahooFinance.egb. We will gather training data for the last 2 years, stopping 60 days short of today.

from java.io import File
from java.util import *
from org.encog.ml.data.market import MarketMLDataSet,MarketDataDescription,TickerSymbol,MarketDataType
from org.encog.ml.data.market.loader import YahooFinanceLoader
from org.encog.util.simple import EncogUtility

INPUT_WINDOW=10;    PREDICT_WINDOW=1 
TICKER =TickerSymbol("AAPL")

loader =YahooFinanceLoader()
market =MarketMLDataSet(loader,INPUT_WINDOW, PREDICT_WINDOW) 
desc=MarketDataDescription(TICKER, MarketDataType.ADJUSTED_CLOSE, True, True) 
market.addDescription(desc)

end = GregorianCalendar(); begin =end.clone() 
begin.add(Calendar.DATE, -60) # Gather training data for the last 2 years, stopping 60 days short of today.
end.add(Calendar.DATE, -60)   # The 60 days will be used to evaluate prediction.
begin.add(Calendar.YEAR, -2)

market.load(begin.getTime(), end.getTime())
market.generate()
print "Fetched records=",market.getRecordCount()

# print input and ideal output
for p in market.getData():
        print p.getInputArray().tolist(), ' ideal=',  p.getIdealArray().tolist()
print "--> save data in a current directory"
EncogUtility.saveEGB(File("./","YahooFinance.egb"), market)

This script also prints input (10 values) and output (one value): A single entry out of 491 will look as this:

[0.00384, 0.00178, 0.006807,
-0.00833, 0.00795, 0.014900,
-0.00409, 0.00220, -0.00312,
-0.00934]  
ideal= [-0.01494]

Now let us build a neural net with 10 inputs, one output and 20 hidden layers. We will train this network for 1 minute and then save the trained network in a file. The script code is given below. <ifauthr>

from java.io import File
from org.encog.util.simple import  EncogUtility

MINUTES = 1  # how many minutes to train
HID1 = 20    # number of hidden layers
HID2 = 0 

# get training data
data=EncogUtility.loadEGB2Memory( File("./","YahooFinance.egb") )
# train network
network = EncogUtility.simpleFeedForward(data.getInputSize(),HID1, HID2,data.getIdealSize(),True)
EncogUtility.trainConsole(network, data,  MINUTES)
print "Final Error: ", network.calculateError(data)
print "Training complete, saving network"
EncogDirectoryPersistence.saveObject(File("marketNetwork.eg"), network)

</ifauth>

Now we read data from YahooFinance again, this time using last 30 days. Then we read trained neural network and perform the prediction based on the last 2 year trend. We compare the actual and predicted value, and print likelihood of our correct prediction:

from java.io import File
from java.util import *
from org.encog.ml.data.market import MarketMLDataSet,MarketDataDescription,TickerSymbol,MarketDataType
from org.encog.ml.data.market.loader import YahooFinanceLoader
from org.encog.util.simple import EncogUtility
from math import  *
from java.text import DecimalFormat
from org.encog.persist import EncogDirectoryPersistence

INPUT_WINDOW=10;    
PREDICT_WINDOW=1 
TICKER =TickerSymbol("AAPL")

# +1 is UP
# -1 is down
def determineDirection(d):
        if (d < 0): return -1;
        else: return  1;
print "Fetching data from Yahoo .. "
loader =YahooFinanceLoader()
data=MarketMLDataSet(loader,INPUT_WINDOW, PREDICT_WINDOW) 
desc=MarketDataDescription(TICKER, MarketDataType.ADJUSTED_CLOSE, True, True) 
data.addDescription(desc)
end=GregorianCalendar() # today 
begin =end.clone()             #  begin 30 days ago
begin.add(Calendar.DATE, -60)
data.load(begin.getTime(), end.getTime())
data.generate()

# get neural net 
network = EncogDirectoryPersistence.loadObject(File("marketNetwork.eg"));
format=DecimalFormat("#0.0000")

correct=0.0;  count=0.0 
print "Print prediction.."
for pair in  data:
       input=pair.getInput()
       actualData = pair.getIdeal()
       predictData = network.compute(input)
       actual = actualData.getData(0)
       predict = predictData.getData(0)
       diff = abs(predict - actual)
       actualDirection = determineDirection(actual)
       predictDirection = determineDirection(predict)
       count= count+1
       if actualDirection == predictDirection:       correct=correct+1
       print  "Day ",count, ":actual=",format.format(actual) ,"(", actualDirection ,") predict=" + format.format(predict),"(",predictDirection,"),diff=",diff
       
percent =correct /count;
print "Direction correct:", correct, "/",count
print "Directional Accuracy:",format.format(percent * 100) ,"%";

The output of the execution of this script will look something like:

Fetching data from Yahoo .. 
Print prediction..
Day  1.0 :actual= -0.0113 ( -1 ) predict=-0.0041 ( -1 ),diff= 0.00723281045594
Day  2.0 :actual= -0.0108 ( -1 ) predict=0.0056 ( 1 ),diff= 0.016323498302
Day  3.0 :actual= 0.0070 ( 1 ) predict=0.0068 ( 1 ),diff= 0.000142392841989
Day  4.0 :actual= 0.0320 ( 1 ) predict=0.0026 ( 1 ),diff= 0.0294366145542
Day  5.0 :actual= 0.0226 ( 1 ) predict=-0.0030 ( -1 ),diff= 0.0256697221103
Day  6.0 :actual= 0.0055 ( 1 ) predict=0.0132 ( 1 ),diff= 0.00767318237177
Day  7.0 :actual= -0.0080 ( -1 ) predict=0.0118 ( 1 ),diff= 0.0198059904802
Day  8.0 :actual= -0.0057 ( -1 ) predict=0.0092 ( 1 ),diff= 0.0148906339578
Day  9.0 :actual= 0.0176 ( 1 ) predict=0.0020 ( 1 ),diff= 0.0156139264763
Day  10.0 :actual= -0.0038 ( -1 ) predict=0.0031 ( 1 ),diff= 0.00693045712649
Day  11.0 :actual= 0.0156 ( 1 ) predict=-0.0033 ( -1 ),diff= 0.0188739030904
Day  12.0 :actual= -0.0018 ( -1 ) predict=0.0065 ( 1 ),diff= 0.00835930166595
Day  13.0 :actual= 0.0022 ( 1 ) predict=0.0024 ( 1 ),diff= 0.000244326149596
Day  14.0 :actual= 0.0065 ( 1 ) predict=-0.0063 ( -1 ),diff= 0.0127469148732
Day  15.0 :actual= 0.0003 ( 1 ) predict=0.0071 ( 1 ),diff= 0.00687984465776
Day  16.0 :actual= 0.0034 ( 1 ) predict=-0.0029 ( -1 ),diff= 0.00624265349531
Day  17.0 :actual= -0.0158 ( -1 ) predict=0.0020 ( 1 ),diff= 0.0177893230083
Day  18.0 :actual= 0.0032 ( 1 ) predict=-0.0046 ( -1 ),diff= 0.00781173668604
Day  19.0 :actual= -0.0172 ( -1 ) predict=0.0009 ( 1 ),diff= 0.0180277119123
Day  20.0 :actual= 0.0514 ( 1 ) predict=-0.0004 ( -1 ),diff= 0.0518078026793
Day  21.0 :actual= -0.0046 ( -1 ) predict=0.0060 ( 1 ),diff= 0.010602114572
Day  22.0 :actual= 0.0057 ( 1 ) predict=0.0051 ( 1 ),diff= 0.000595736696516
Day  23.0 :actual= 0.0154 ( 1 ) predict=0.0064 ( 1 ),diff= 0.00904954951323
Day  24.0 :actual= 0.0124 ( 1 ) predict=0.0152 ( 1 ),diff= 0.00284543760372
Day  25.0 :actual= -0.0018 ( -1 ) predict=-0.0040 ( -1 ),diff=
0.00223833948343
Day  26.0 :actual= 0.0092 ( 1 ) predict=0.0017 ( 1 ),diff= 0.0074896027783
Day  27.0 :actual= 0.0129 ( 1 ) predict=0.0012 ( 1 ),diff= 0.0116513741193
Day  28.0 :actual= 0.0149 ( 1 ) predict=0.0018 ( 1 ),diff= 0.0131001498602
Day  29.0 :actual= -0.0089 ( -1 ) predict=0.0062 ( 1 ),diff= 0.0151645127572
Day  30.0 :actual= -0.0006 ( -1 ) predict=-0.0028 ( -1 ),diff=
0.00226212010508
Day  31.0 :actual= -0.0020 ( -1 ) predict=-0.0015 ( -1 ),diff=
0.000477980160665
Day  32.0 :actual= -0.0142 ( -1 ) predict=-0.0026 ( -1 ),diff= 0.0116349624231
Direction correct: 25.0 / 32.0
Directional Accuracy: 63.1250 %