DMelt:Finance/Market Prediction
From HandWiki
Member
Market prediction
In Section Neural Network we discussed how to build a neural net based on backpropogation algorithm. Now we will show how to use this neural network approach to make predictions of financial market market.
Using Joone
Let us consider Joone Neural net for forecasting market predictions using a generated time series in the form cos(x)*sin(x)+Gaussian noise.
from org.joone.engine import NeuralNetListener,TanhLayer,DelayLayer
from org.joone.engine import SigmoidLayer,FullSynapse,Monitor
from org.joone.io import FileInputSynapse,FileOutputSynapse
from org.joone.engine.learning import TeachingSynapse
from org.joone.net import NeuralNet
from java.lang import System
from java.io import *
from jhplot import *
from java.awt import *
import math
from java.util import Random
class joone(NeuralNetListener):
mills=0
epochs=1000
fileName=""
nnet = NeuralNet()
input = DelayLayer()
hidden = SigmoidLayer()
output = TanhLayer()
# a function to help connect layers
def connect(self,layer1, syn, layer2):
layer1.addOutputSynapse(syn)
layer2.addInputSynapse(syn)
def createDataSet(self,fileName,firstRow, lastRow, advColSel):
fInput = FileInputSynapse()
fInput.setInputFile(File(fileName))
fInput.setFirstRow(firstRow)
fInput.setLastRow(lastRow)
fInput.setAdvancedColumnSelector(advColSel)
return fInput
def createNet(self,fileName,epochs,trainingPatterns,temporalWindow):
self.epochs=epochs
print "Running NN for ", epochs, " epochs"
self.fileName=fileName
self.input.setTaps(temporalWindow-1)
self.input.setRows(1)
self.hidden.setRows(15)
self.output.setRows(1)
self.connect(self.input,FullSynapse(), self.hidden)
self.connect(self.hidden,FullSynapse(), self.output)
self.input.addInputSynapse(self.createDataSet(self.fileName, 1, trainingPatterns, "1"))
trainer = TeachingSynapse()
trainer.setDesired(self.createDataSet(self.fileName, 2, trainingPatterns+1, "1"))
self.output.addOutputSynapse(trainer)
self.nnet.addLayer(self.input, NeuralNet.INPUT_LAYER)
self.nnet.addLayer(self.hidden, NeuralNet.HIDDEN_LAYER)
self.nnet.addLayer(self.output, NeuralNet.OUTPUT_LAYER)
self.mills = System.currentTimeMillis()
self.nnet.randomize(0.5)
def train(self):
mon = self.nnet.getMonitor()
mon.setLearningRate(0.2)
mon.setMomentum(0.7)
mon.setTrainingPatterns(trainingPatterns)
mon.setTotCicles(epochs)
mon.setPreLearning(temporalWindow)
mon.setLearning(True)
mon.addNeuralNetListener(self)
self.nnet.start()
mon.Go()
self.nnet.join()
def interrogate(self,outputFile):
mon = self.nnet.getMonitor()
self.input.removeAllInputs()
startRow = trainingPatterns - temporalWindow
self.input.addInputSynapse(self.createDataSet(self.fileName, startRow+1, startRow+40, "1"))
self.output.removeAllOutputs()
fOutput = FileOutputSynapse()
fOutput.setFileName(outputFile)
self.output.addOutputSynapse(fOutput)
mon.setTrainingPatterns(40)
mon.setTotCicles(1)
mon.setLearning(False)
self.nnet.start()
mon.Go()
self.nnet.join()
def netStopped(self,e):
mon =e.getSource()
if (mon.isLearning()):
epoch = mon.getTotCicles() - mon.getCurrentCicle()
print "Epoch:",epoch," last RMSE=",mon.getGlobalError()
else:
delay = System.currentTimeMillis() - self.mills
print "Training finished after ",delay," ms"
def cicleTerminated(self,e):
mon = e.getSource()
epoch = mon.getTotCicles() - mon.getCurrentCicle()
if ((epoch > 0) and ((epoch % 100) == 0)):
print "Epoch:",epoch," RMSE=",mon.getGlobalError()
def netStarted(self,e):
pass
def errorChanged(self,e):
pass
def netStoppedError(self,e):
pass
r=Random()
print "Create a file joone_timeseries.txt with time series cos(x)*sin(x)+noise"
f = open("joone_timeseries.txt", "w")
for x in range(0,1000):
yEst = 0.9*math.cos(x*0.4)*math.sin(0.7*x)+0.02*r.nextGaussian() # cos(x*0.3)*sim(x)+noise
f.write( str(yEst) + "\n" )
f.close()
# get input file with TimeSeries
fileName = "joone_timeseries.txt"
print "Downloading .. "+fileName
# Web.get("http://datamelt.org/examples/data/"+fileName)
trainingPatterns = 200
epochs = 10000
temporalWindow = 20
c1 = HPlot("show data")
c1.visible()
c1.setRangeX(0,1100)
c1.setNameX("time")
c1.setNameY("price")
c1.setMarginLeft(90)
# show data
dataIN = [line.strip() for line in open(fileName, 'r')]
print "Total data size=",len(dataIN), " For learning=",trainingPatterns
p1=P1D("Time series")
p1.setDrawLine(True)
p1.setPenWidth(1)
for i in range(len(dataIN)):
p1.add(i,float(dataIN[i]))
c1.draw(p1)
# run NN for predictions
ts=joone()
ts.createNet(fileName,epochs,trainingPatterns,temporalWindow)
print "Training..."
ts.train()
ts.interrogate("results1.txt")
ts.interrogate("results2.txt")
# show predictions
data = [line.strip() for line in open("results2.txt", 'r')]
p2=P1D("Prediction")
p2.setColor(Color.red)
p2.setDrawLine(True)
for i in range(len(data)):
p2.add(len(dataIN)+i,float(data[i]))
c1.draw(p2)
print "Done."
Here is the output image that shows the original time series and the predicted trends (n red).
As you can see, the neural net correctly predicts the expected behavior.
Market prediction
Now we consider another practical example. First, we will fetch data from Yahoo and store this data in a file: YahooFinance.egb. We will gather training data for the last 2 years, stopping 60 days short of today.
from java.io import File
from java.util import *
from org.encog.ml.data.market import MarketMLDataSet,MarketDataDescription,TickerSymbol,MarketDataType
from org.encog.ml.data.market.loader import YahooFinanceLoader
from org.encog.util.simple import EncogUtility
INPUT_WINDOW=10; PREDICT_WINDOW=1
TICKER =TickerSymbol("AAPL")
loader =YahooFinanceLoader()
market =MarketMLDataSet(loader,INPUT_WINDOW, PREDICT_WINDOW)
desc=MarketDataDescription(TICKER, MarketDataType.ADJUSTED_CLOSE, True, True)
market.addDescription(desc)
end = GregorianCalendar(); begin =end.clone()
begin.add(Calendar.DATE, -60) # Gather training data for the last 2 years, stopping 60 days short of today.
end.add(Calendar.DATE, -60) # The 60 days will be used to evaluate prediction.
begin.add(Calendar.YEAR, -2)
market.load(begin.getTime(), end.getTime())
market.generate()
print "Fetched records=",market.getRecordCount()
# print input and ideal output
for p in market.getData():
print p.getInputArray().tolist(), ' ideal=', p.getIdealArray().tolist()
print "--> save data in a current directory"
EncogUtility.saveEGB(File("./","YahooFinance.egb"), market)
This script also prints input (10 values) and output (one value): A single entry out of 491 will look as this:
[0.00384, 0.00178, 0.006807,
-0.00833, 0.00795, 0.014900,
-0.00409, 0.00220, -0.00312,
-0.00934]
ideal= [-0.01494]
Now let us build a neural net with 10 inputs, one output and 20 hidden layers.
We will train this network for 1 minute and then save the trained network in a file.
The script code is given below.
<ifauthr>
from java.io import File
from org.encog.util.simple import EncogUtility
MINUTES = 1 # how many minutes to train
HID1 = 20 # number of hidden layers
HID2 = 0
# get training data
data=EncogUtility.loadEGB2Memory( File("./","YahooFinance.egb") )
# train network
network = EncogUtility.simpleFeedForward(data.getInputSize(),HID1, HID2,data.getIdealSize(),True)
EncogUtility.trainConsole(network, data, MINUTES)
print "Final Error: ", network.calculateError(data)
print "Training complete, saving network"
EncogDirectoryPersistence.saveObject(File("marketNetwork.eg"), network)
</ifauth>
Now we read data from YahooFinance again, this time using last 30 days. Then we read trained neural network and perform the prediction based on the last 2 year trend. We compare the actual and predicted value, and print likelihood of our correct prediction:
from java.io import File
from java.util import *
from org.encog.ml.data.market import MarketMLDataSet,MarketDataDescription,TickerSymbol,MarketDataType
from org.encog.ml.data.market.loader import YahooFinanceLoader
from org.encog.util.simple import EncogUtility
from math import *
from java.text import DecimalFormat
from org.encog.persist import EncogDirectoryPersistence
INPUT_WINDOW=10;
PREDICT_WINDOW=1
TICKER =TickerSymbol("AAPL")
# +1 is UP
# -1 is down
def determineDirection(d):
if (d < 0): return -1;
else: return 1;
print "Fetching data from Yahoo .. "
loader =YahooFinanceLoader()
data=MarketMLDataSet(loader,INPUT_WINDOW, PREDICT_WINDOW)
desc=MarketDataDescription(TICKER, MarketDataType.ADJUSTED_CLOSE, True, True)
data.addDescription(desc)
end=GregorianCalendar() # today
begin =end.clone() # begin 30 days ago
begin.add(Calendar.DATE, -60)
data.load(begin.getTime(), end.getTime())
data.generate()
# get neural net
network = EncogDirectoryPersistence.loadObject(File("marketNetwork.eg"));
format=DecimalFormat("#0.0000")
correct=0.0; count=0.0
print "Print prediction.."
for pair in data:
input=pair.getInput()
actualData = pair.getIdeal()
predictData = network.compute(input)
actual = actualData.getData(0)
predict = predictData.getData(0)
diff = abs(predict - actual)
actualDirection = determineDirection(actual)
predictDirection = determineDirection(predict)
count= count+1
if actualDirection == predictDirection: correct=correct+1
print "Day ",count, ":actual=",format.format(actual) ,"(", actualDirection ,") predict=" + format.format(predict),"(",predictDirection,"),diff=",diff
percent =correct /count;
print "Direction correct:", correct, "/",count
print "Directional Accuracy:",format.format(percent * 100) ,"%";
The output of the execution of this script will look something like:
Fetching data from Yahoo .. Print prediction.. Day 1.0 :actual= -0.0113 ( -1 ) predict=-0.0041 ( -1 ),diff= 0.00723281045594 Day 2.0 :actual= -0.0108 ( -1 ) predict=0.0056 ( 1 ),diff= 0.016323498302 Day 3.0 :actual= 0.0070 ( 1 ) predict=0.0068 ( 1 ),diff= 0.000142392841989 Day 4.0 :actual= 0.0320 ( 1 ) predict=0.0026 ( 1 ),diff= 0.0294366145542 Day 5.0 :actual= 0.0226 ( 1 ) predict=-0.0030 ( -1 ),diff= 0.0256697221103 Day 6.0 :actual= 0.0055 ( 1 ) predict=0.0132 ( 1 ),diff= 0.00767318237177 Day 7.0 :actual= -0.0080 ( -1 ) predict=0.0118 ( 1 ),diff= 0.0198059904802 Day 8.0 :actual= -0.0057 ( -1 ) predict=0.0092 ( 1 ),diff= 0.0148906339578 Day 9.0 :actual= 0.0176 ( 1 ) predict=0.0020 ( 1 ),diff= 0.0156139264763 Day 10.0 :actual= -0.0038 ( -1 ) predict=0.0031 ( 1 ),diff= 0.00693045712649 Day 11.0 :actual= 0.0156 ( 1 ) predict=-0.0033 ( -1 ),diff= 0.0188739030904 Day 12.0 :actual= -0.0018 ( -1 ) predict=0.0065 ( 1 ),diff= 0.00835930166595 Day 13.0 :actual= 0.0022 ( 1 ) predict=0.0024 ( 1 ),diff= 0.000244326149596 Day 14.0 :actual= 0.0065 ( 1 ) predict=-0.0063 ( -1 ),diff= 0.0127469148732 Day 15.0 :actual= 0.0003 ( 1 ) predict=0.0071 ( 1 ),diff= 0.00687984465776 Day 16.0 :actual= 0.0034 ( 1 ) predict=-0.0029 ( -1 ),diff= 0.00624265349531 Day 17.0 :actual= -0.0158 ( -1 ) predict=0.0020 ( 1 ),diff= 0.0177893230083 Day 18.0 :actual= 0.0032 ( 1 ) predict=-0.0046 ( -1 ),diff= 0.00781173668604 Day 19.0 :actual= -0.0172 ( -1 ) predict=0.0009 ( 1 ),diff= 0.0180277119123 Day 20.0 :actual= 0.0514 ( 1 ) predict=-0.0004 ( -1 ),diff= 0.0518078026793 Day 21.0 :actual= -0.0046 ( -1 ) predict=0.0060 ( 1 ),diff= 0.010602114572 Day 22.0 :actual= 0.0057 ( 1 ) predict=0.0051 ( 1 ),diff= 0.000595736696516 Day 23.0 :actual= 0.0154 ( 1 ) predict=0.0064 ( 1 ),diff= 0.00904954951323 Day 24.0 :actual= 0.0124 ( 1 ) predict=0.0152 ( 1 ),diff= 0.00284543760372 Day 25.0 :actual= -0.0018 ( -1 ) predict=-0.0040 ( -1 ),diff= 0.00223833948343 Day 26.0 :actual= 0.0092 ( 1 ) predict=0.0017 ( 1 ),diff= 0.0074896027783 Day 27.0 :actual= 0.0129 ( 1 ) predict=0.0012 ( 1 ),diff= 0.0116513741193 Day 28.0 :actual= 0.0149 ( 1 ) predict=0.0018 ( 1 ),diff= 0.0131001498602 Day 29.0 :actual= -0.0089 ( -1 ) predict=0.0062 ( 1 ),diff= 0.0151645127572 Day 30.0 :actual= -0.0006 ( -1 ) predict=-0.0028 ( -1 ),diff= 0.00226212010508 Day 31.0 :actual= -0.0020 ( -1 ) predict=-0.0015 ( -1 ),diff= 0.000477980160665 Day 32.0 :actual= -0.0142 ( -1 ) predict=-0.0026 ( -1 ),diff= 0.0116349624231 Direction correct: 25.0 / 32.0 Directional Accuracy: 63.1250 %
