FX automatic trading system made with python and genetic algorithm part 1 Forex automatic trading with genetic algorithm Part 2 Evolving trading AI implementation
I would like to write about the Oanda API that I used when I made the automatic trading system and the function of placing buy and sell orders.
It was built using 5 types of APIs. For details, refer to Oanda's API Document OAuth2 Token can be issued on the Web by opening an account and logging in. Set the token to Authorization: Bearer ******** on the curl command Header.
"""
01.Account information API
curl -H "Authorization: Bearer ********" https://api-
fxtrade.oanda.com/v1/accounts/12345
"""
{
"accountId" : 12345,
"accountName" : "Primary",
"balance" : 123456.5765,
"unrealizedPl" : 36.8816,
"realizedPl" : 235.5839,
"marginUsed" : 123456.154,
"marginAvail" : 123456.3041,
"openTrades" : 20,
"openOrders" : 0,
"marginRate" : 0.04,
"accountCurrency" : "JPY"
}
"""
02.Ordering API
curl -H "Authorization: Bearer ********" -X POST -d "instrument=EUR_USD&units=2&side=sell&type=market" "https://api-fxtrade.oanda.com/v1/accounts/12345/orders"
"""
{
"instrument" : "EUR_USD",
"time" : "2013-12-06T20:36:06Z", // Time that order was executed
"price" : 1.37041, // Trigger price of the order
"tradeOpened" : {
"id" : 175517237, // Order id
"units" : 1000, // Number of units
"side" : "buy", // Direction of the order
"takeProfit" : 0, // The take-profit associated with the Order, if any
"stopLoss" : 0, // The stop-loss associated with the Order, if any
"trailingStop" : 0 // The trailing stop associated with the rrder, if any
},
"tradesClosed" : [],
"tradeReduced" : {}
}
"""
03.Get the current position list
curl -H "Authorization: Bearer ####" https://api-fxtrade.oanda.com/v1/accounts/######/positions
"""
{
"positions" : [
{
"instrument" : "AUD_USD",
"units" : 100,
"side" : "sell",
"avgPrice" : 0.78216
},
{
"instrument" : "GBP_USD",
"units" : 100,
"side" : "sell",
"avgPrice" : 1.49128
},
{
"instrument" : "USD_JPY",
"units" : 850,
"side" : "buy",
"avgPrice" : 119.221
}
}
"""
04.Get market rate
curl -H "Authorization: Bearer ********" -X GET "https://api-fxtrade.oanda.com/v1/prices?instruments=EUR_USD%2CUSD_JPY%2CEUR_CAD"
"""
{
"prices" : [
{
"instrument" : "EUR_USD",
"time" : "2015-03-13T20:59:58.165668Z",
"bid" : 1.04927,
"ask" : 1.04993,
"status" : "halted"
},
{
"instrument" : "USD_JPY",
"time" : "2015-03-13T20:59:58.167818Z",
"bid" : 121.336,
"ask" : 121.433,
"status" : "halted"
},
{
"instrument" : "EUR_CAD",
"time" : "2015-03-13T20:59:58.165465Z",
"bid" : 1.34099,
"ask" : 1.34225,
"status" : "halted"
}
]
}
"""
05.API to check the order status of buy and sell orders
curl -H "Authorization: Bearer ************" https://api-fxtrade.oanda.com/v1/accounts/****/transactions
"""
{
"id" : 1789536248,
"accountId" : *****,
"time" : "2015-03-20T12:17:06.000000Z",
"type" : "TAKE_PROFIT_FILLED",
"tradeId" : 1789531428,
"instrument" : "USD_JPY",
"units" : 1,
"side" : "sell",
"price" : 121.017,
"pl" : 0.02,
"interest" : 0,
"accountBalance" : 33299999.1173
},
{
"id" : 1789560748,
"accountId" : *****,
"time" : "2015-03-20T12:49:38.000000Z",
"type" : "STOP_LOSS_FILLED",
"tradeId" : 1789500620,
"instrument" : "USD_JPY",
"units" : 100,
"side" : "sell",
"price" : 120.899,
"pl" : -18.6,
"interest" : 0,
"accountBalance" : 33299980.3023
}
This is an implementation example of the price acquisition API. Web access with urllib request. The response will be returned in Json, so I'll do my best to write the parser class. When the spread opens (the market becomes rough and the difference between the bid price and the sell price opens), I tried to incorporate a function that does not place an order when the rate is unfavorable.
■ Sample code accessing the sandbox API http://api-sandbox.oanda.com/v1/prices?instruments=EUR_USD%2CUSD_JPY%2CGBP_USD%2CAUD_USD
price_api.py
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import unicode_literals
import urllib
import requests
import time
import ujson
import pytz
from enum import Enum
class CurrencyPair(Enum):
EUR_USD = 1
USD_JPY = 2
GBP_USD = 3
AUD_USD = 4
class OandaAPIMode(Enum):
PRODUCTION = 1
DEVELOP = 2
SANDBOX = 3
DUMMY = 4
@property
def url_base(self):
api_base_url_dict = {
OandaAPIMode.PRODUCTION: 'https://api-fxtrade.oanda.com/',
OandaAPIMode.DEVELOP: 'https://api-fxpractice.oanda.com/',
OandaAPIMode.SANDBOX: 'http://api-sandbox.oanda.com/',
}
return api_base_url_dict.get(self)
@property
def headers(self):
"""
:rtype : dict
"""
_base = {
'Accept-Encoding': 'identity, deflate, compress, gzip',
'Accept': '*/*', 'User-Agent': 'python-requests/1.2.0',
'Content-type': 'application/x-www-form-urlencoded',
}
if self == OandaAPIMode.SANDBOX:
return _base
if self == OandaAPIMode.DEVELOP:
_base['Authorization'] = 'Bearer {}'.format(get_password('OandaRestAPITokenDemo'))
return _base
if self == OandaAPIMode.PRODUCTION:
_base['Authorization'] = 'Bearer {}'.format(get_password('OandaRestAPIToken'))
return _base
raise ValueError
class OandaServiceUnavailableError(Exception):
"""
Error during service stop
"""
pass
class OandaInternalServerError(Exception):
"""
Error during service stop
"""
pass
class OandaAPIBase(object):
mode = None
class Meta(object):
abstract = True
def __init__(self, mode):
"""
:param mode: OandaAPIMode
"""
self.mode = mode
def requests_api(self, url, payload=None):
#If communication fails, retry up to 3 times
for x in xrange(3):
response = None
try:
if payload:
response = requests.post(url, headers=self.mode.headers, data=payload)
else:
response = requests.get(url, headers=self.mode.headers)
assert response.status_code == 200, response.status_code
print 'API_Access: {}'.format(url)
data = ujson.loads(response.text)
self.check_json(data)
return data
except Exception as e:
if response is None:
raise
if response.text:
if str("Service Unavailable") in str(response.text):
raise OandaServiceUnavailableError
if str("An internal server error occurred") in str(response.text):
raise OandaInternalServerError
time.sleep(3)
if x >= 2:
raise ValueError, response.text
raise
def check_json(self, data):
raise NotImplementedError
class PriceAPI(OandaAPIBase):
"""
Rate confirmation API
%2C is a comma
curl -H "Authorization: Bearer ********" -X GET "https://api-fxtrade.oanda.com/v1/prices?instruments=EUR_USD%2CUSD_JPY%2CEUR_CAD"
{
"prices" : [
{
"instrument" : "EUR_USD",
"time" : "2015-03-13T20:59:58.165668Z",
"bid" : 1.04927,
"ask" : 1.04993,
"status" : "halted"
},
{
"instrument" : "USD_JPY",
"time" : "2015-03-13T20:59:58.167818Z",
"bid" : 121.336,
"ask" : 121.433,
"status" : "halted"
},
{
"instrument" : "EUR_CAD",
"time" : "2015-03-13T20:59:58.165465Z",
"bid" : 1.34099,
"ask" : 1.34225,
"status" : "halted"
}
]
}
:rtype : dict of PriceAPIModel
"""
url_base = '{}v1/prices?'
def get_all(self):
instruments = ','.join([x.name for x in CurrencyPair])
instruments = urllib.quote(instruments, '')
url = self.url_base.format(self.mode.url_base)
url += 'instruments={}'.format(instruments)
data = self.requests_api(url)
d = {}
for price in data.get('prices'):
price_model = PriceAPIModel(price)
d[price_model.currency_pair] = price_model
return d
def check_json(self, data):
assert 'prices' in data
class PriceAPIModel(object):
"""
Class that parses the rate confirmation API
"""
instrument = None
time = None
bid = None #Tick at the time of SELL
ask = None #Tick at the time of BUY
status = None
def __init__(self, price):
self.ask = float(price.get('ask'))
self.bid = float(price.get('bid'))
self.time = parse_time(price.get('time'))
self.instrument = str(price.get('instrument'))
if 'status' in price:
self.status = str(price.get('status'))
self._check(price)
def _check(self, price):
if 'ask' not in price:
raise ValueError
if 'bid' not in price:
raise ValueError
if 'time' not in price:
raise ValueError
if 'instrument' not in price:
raise ValueError
self.currency_pair
@property
def currency_pair(self):
"""
:rtype : CurrencyPair
"""
for pair in CurrencyPair:
if str(pair.name) == str(self.instrument):
return pair
raise ValueError
@property
def is_maintenance(self):
"""
True if during maintenance
:rtype : bool
"""
if self.status is None:
return False
if str(self.status) == str('halted'):
return True
return False
def is_active(self):
"""
True if valid rate
:rtype : bool
"""
#Not in maintenance
if self.is_maintenance:
return False
#Allows up to 4 ticks normal rate
if self.cost_tick > 4:
return False
return True
def get_password(key):
"""
Respond to password
"""
return "12345"
def parse_time(time_text):
"""
Convert a string to UTC.
Example)
2015-02-22T15:00:00.000000Z
:param time_text: char
:rtype : datetime
"""
import datetime
t = time_text
utc = datetime.datetime(int(t[0:4]),
int(t[5:7]),
int(t[8:10]),
int(t[11:13]),
int(t[14:16]),
int(t[17:19]),
int(t[20:26]), tzinfo=pytz.utc)
return utc
def utc_to_jst(utc):
return utc.astimezone(pytz.timezone('Asia/Tokyo'))
print "START"
# API_Access
price_group_dict = PriceAPI(OandaAPIMode.SANDBOX).get_all()
#Print the result
for key in price_group_dict:
pair = key
price = price_group_dict[key]
print "Pair:{} bid:{} ask:{} time:{}".format(pair.name, price.bid, price.ask, price.time)
print "FINISH"
Execution result
>python price_api.py
START
API_Access: http://api-sandbox.oanda.com/v1/prices?instruments=EUR_USD%2CUSD_JPY%2CGBP_USD%2CAUD_USD
Pair:USD_JPY bid:120.243 ask:120.293 time:2015-09-23 21:07:30.363800+00:00
Pair:AUD_USD bid:0.70049 ask:0.70066 time:2015-09-23 21:07:30.367431+00:00
Pair:EUR_USD bid:1.13916 ask:1.1393 time:2015-10-13 07:16:53.931425+00:00
Pair:GBP_USD bid:1.52455 ask:1.5248 time:2015-09-23 21:07:30.362069+00:00
FINISH
Unexpected errors often occur in production systems. Since the Oanda API is simple, I think it will be easier to write the library yourself later. If you want to use the library, it is better to use the library considering that the response of the Saturday and Sunday API will stop.
When 100 AI buys and sells dollar yen at the timing when the 5-minute bar is updated, if buy is 60 and sell is 40, the overlapping 40 parts will be denominated in both and will be lost in the commission. As a countermeasure for the double-decker problem, we will optimize buying and selling in bulk orders.
■ Sample case Minimize trading fees when AI1-6 place an order as follows. It is assumed that one transaction fee is 100 yen.
AI Name | Order | bid | Limit | StopLimit |
---|---|---|---|---|
AI.001 | Buy | 120.00 | 120.50 | 119.50 |
AI.002 | Buy | 120.00 | 120.70 | 119.50 |
AI.003 | Buy | 120.00 | 120.50 | 119.50 |
AI.004 | Buy | 120.00 | 120.50 | 119.00 |
AI.005 | Sell | 120.00 | 119.00 | 120.50 |
AI.006 | Sell | 120.00 | 118.50 | 120.50 |
■ Method 1. At 120.00 yen, place an order for 4 Buy and 2 Sell (trading fee 600 yen) After that, the market price fluctuates and all 6 orders are consumed (trading fee 600 yen). 1200 yen in total
■ Method 2. At the time of 120.00 yen, the orders of AI1 to 6 are put together and the dollar yen 120.00 Buy 2 orders (trading fee 200 yen) After that, the market price fluctuates and all 6 orders are consumed (trading fee 600 yen). 800 yen in total
Since Oanda is a double-decker system that does not have a position, I decided to buy and sell using the solution 2.
In the solution 1 method, you can set Limit and Stop Limit for each order and place an order, so once you place the order, Oanda will automatically settle the payment, but if you use the solution 2 method, the order unit will be different, so you can place an order. You also need to place an order for payment from your own program.
What if the payment program stops with an error? The system will continue to place orders, but the positions will not be closed. Leverage of your account will continue to rise. To prevent this from happening, trading programs are required to have high availability. I also installed Call notification function when an error occurs.
To achieve high availability, it is easy to process the program with supervisord. Create open and close commands for positions (buy and sell orders) in Python and use supervisord to launch the process. Similar functions can be achieved with Cron and while True, but they have many disadvantages and were not adopted.
method | Demerit |
---|---|
cron | Immediate re-execution is not possible after the command ends |
while True | Memory leak occurs, error cannot be automatically recovered when error occurs |
supervisord | Hassle to install, hassle to deploy |
order_open.py
# -*- coding: utf-8 -*-
#It's a pseudo-language because it's empowered. Maybe it doesn't work. sorry
from __future__ import absolute_import
from __future__ import unicode_literals
import datetime
from django.core.management import BaseCommand
import time
import sys
import pytz
from requests import ConnectionError
ACCOUNT_ID = 12345
class CustomBaseCommand(BaseCommand):
class Meta(object):
abstract = True
def echo(self, txt):
d = datetime.datetime.today()
print d.strftime("%Y-%m-%d %H:%M:%S") + ':' + txt
class Command(CustomBaseCommand):
"""
Place an order using BoardAI
"""
CACHE_PREV_RATES = {}
def handle(self, *args, **options):
try:
self.run()
except OandaServiceUnavailableError:
#During maintenance on Saturdays and Sundays
self.echo("ServiceUnavailableError")
time.sleep(60)
except OandaInternalServerError:
#During maintenance on Saturdays and Sundays
self.echo("OandaInternalServerError")
time.sleep(60)
except ConnectionError:
time.sleep(60)
#Stop for 3 seconds
time.sleep(3)
def run(self):
#Margin check
account = AccountAPI(OandaAPIMode.PRODUCTION, ACCOUNT_ID).get_all()
if int(account.get('marginAvail')) < 10000:
self.echo('MARGIN EMPTY!! marginAvail:{}'.format(int(account.get('marginAvail'))))
time.sleep(3000)
#take price
price_group = PriceAPI(OandaAPIMode.PRODUCTION).get_all()
#AI instance generation
order_group = []
ai_board_group = AIBoard.get_all()
#Temporary order
now = datetime.datetime.now(tz=pytz.utc)
for ai_board in ai_board_group:
order = self.pre_order(ai_board, price_group, now)
if order:
order_group.append(order)
#Order
self.order(order_group, price_group)
def pre_order(self, ai_board, price_group, now):
"""
Return True after placing an order
:param ai_board: AIBoard
:param price_group: dict of PriceAPIModel
:param now: datetime
:rtype : bool
"""
ai = ai_board.get_ai_instance()
price = price_group.get(ai.currency_pair, None)
#Is the price normal?
if price is None:
return None
if not price.is_active():
# print 'price not active'
return None
#Rate is not normal
prev_rates = self.get_prev_rates(ai.currency_pair, Granularity.H1)
if not prev_rates:
return None
prev_rate = prev_rates[-1]
#Deviation from the previous rate within 3 minutes
if now - prev_rate.start_at > datetime.timedelta(seconds=60 * 3):
# print 'ORDER TIME IS OVER'
return None
#Purchase limit by number of positions and purchase limit by time
if not ai_board.can_order(prev_rate):
# print 'TIME OR POSITION LIMIT'
return None
#Purchase decision Similar to AI simulation, buy and sell is decided only for mid
order_ai = ai.get_order_ai(prev_rates, price.mid, price.time, is_production=True)
if order_ai is None:
return None
if order_ai.order_type == OrderType.WAIT:
return None
#Temporary order firing
order = Order.pre_open(ai_board, order_ai, price, prev_rate.start_at)
return order
def order(self, order_group, price_group):
"""
Take a summary of the pre-order and actually place the order
:param order_group: list of Order
:param price_group: dict of PriceAPIModel
:return:
"""
#Take a summary
order_dict = {x: 0 for x in CurrencyPair}
for order in order_group:
if order.buy:
order_dict[order.currency_pair] += order.units
else:
order_dict[order.currency_pair] -= order.units
print order_dict
#Order
api_response_dict = {}
for key in order_dict:
if order_dict[key] == 0:
print '{}:NO ORDER'.format(key)
continue
print '{}:ORDER!'.format(key)
units = order_dict[key]
api_response_dict[key] = OrdersAPI(OandaAPIMode.PRODUCTION, ACCOUNT_ID).post(key, units, tag='open')
#DB update
for order in order_group:
order.open(price_group.get(order.currency_pair))
def get_prev_rates(self, currency_pair, granularity):
key = self._get_key(currency_pair, granularity)
print key
r = self.CACHE_PREV_RATES.get(key)
if r:
return r
prev_rates = CurrencyPairToTable.get_table(currency_pair, granularity).get_new_record_by_count(10000)
self.CACHE_PREV_RATES[key] = prev_rates
return prev_rates
def _get_key(self, currency_pair, granularity):
return 'RATE:{}:{}'.format(currency_pair.value, granularity.value)
order_close.py
# -*- coding: utf-8 -*-
#It's a pseudo-language because it's empowered. Maybe it doesn't work. sorry
from __future__ import absolute_import
from __future__ import unicode_literals
import datetime
from django.core.management import BaseCommand
import time
import pytz
from requests import ConnectionError
class Command(CustomBaseCommand):
"""
Close using account history
"""
CACHE_PREV_RATES = {}
def handle(self, *args, **options):
print '********************************'
self.echo('close start')
self.check_kill_switch()
try:
self.run()
except OandaServiceUnavailableError:
#During maintenance on Saturdays and Sundays
self.echo("ServiceUnavailableError")
time.sleep(60)
except OandaInternalServerError:
#During maintenance on Saturdays and Sundays
self.echo("OandaInternalServerError")
time.sleep(60)
except ConnectionError:
time.sleep(60)
except Exception as e:
self.critical_error('close', e)
time.sleep(3)
def run(self):
orders = Order.get_open()
#take price
price_group = PriceAPI(OandaAPIMode.PRODUCTION).get_all()
#Take a summary
order_dict = {x: 0 for x in CurrencyPair}
for order in orders:
if order.can_close(price_group[order.currency_pair]):
if order.buy:
#Reverse buying and selling
print order.currency_pair, order.units
order_dict[order.currency_pair] -= order.units
print order_dict[order.currency_pair]
else:
#Reverse buying and selling
print order.currency_pair, order.units
order_dict[order.currency_pair] += order.units
print order_dict[order.currency_pair]
print order_dict
#Order
api_response_dict = {}
for key in order_dict:
if order_dict[key] == 0:
print '{}:NO ORDER'.format(key)
continue
print '{}:ORDER'.format(key)
units = order_dict[key]
api_response_dict[key] = OrdersAPI(OandaAPIMode.PRODUCTION, 12345).post(key, units, tag='close')
#Record
for order in orders:
if order.can_close(price_group[order.currency_pair]):
order.close(price_group.get(order.currency_pair))
I tried to build a calculation cluster of genetic algorithm with oleore specifications. The worker makes a command to calculate with a genetic algorithm in python and starts it with supervisord.
Was it profitable to buy and sell automatically after all?
Month | Trading profit | Server maintenance fee |
---|---|---|
March 2015 | +50 000 | 10 000 |
April 2015 | +20,000 | 10 000 |
May 2015 | -70,000 | 10 000 |
June 2015 | -50 000 | 10 000 |
July 2015 | Stop the automated trading system and ETF:Get the full amount into 1557 | - |
Thank you for staying with us until the end. The development of the automatic trading system was very interesting. Please try it.
FX automatic trading system made with python and genetic algorithm part 1 Forex automatic trading with genetic algorithm Part 2 Evolving trading AI implementation Forex automatic trading with genetic algorithm Part 3 Actual trading with Oanda API
Recommended Posts