www.pudn.com > JSystemTrader.zip > BackDataDownloader.java
package com.jsystemtrader.backdata;
import java.io.*;
import java.text.*;
import java.util.*;
import java.util.prefs.*;
import com.ib.client.*;
import com.jsystemtrader.platform.*;
import com.jsystemtrader.util.*;
/**
* Retrieves historical data for a specified security and saves it to a text file.
*/
public class BackDataDownloader extends Thread implements ErrorListener {
private static final long MAX_FREQUENCY_MILLIS = 2500;
private static final int MAX_REQUESTS_IN_HMDS_SESSION = 60;
private static final int MAX_WAITING_TIME = 3 * 60 * 1000; // 3 minutes to wait for HMDS session expiration
private static final int REQUEST_ID = 1;
private static final String REQUEST_COUNTER = "RequestCounter";
private static final String lineSep = System.getProperty("line.separator");
private static final Preferences preferences = Preferences.systemNodeForPackage(BackDataDownloader.class);
private final Trader trader;
private final HTMLLog eventlogger;
private final String fileName, barSize;
private final boolean rthOnly;
private final QuoteHistory qh;
private List priceBars = new ArrayList ();
private final BackDataDialog backDataDialog;
private PrintWriter writer;
private final Contract contract;
private boolean firstBarReached, hasSessionExpired, isCancelled;
private Calendar firstDate, lastDate;
/** Keeps track of the number of historical requests in the current HMDS session */
private static int requestCounter;
// static initializer
static {
// request counter is persistent from one JSystemTrader run to another
requestCounter = preferences.getInt(REQUEST_COUNTER, 0);
}
public BackDataDownloader(BackDataDialog backDataDialog, Contract contract, String barSize, boolean rthOnly,
String fileName) throws JSystemTraderException {
this.backDataDialog = backDataDialog;
this.contract = contract;
this.barSize = barSize;
this.rthOnly = rthOnly;
this.fileName = fileName;
setPeriodBoundaries();
eventlogger = Account.getLogger();
trader = Account.getTrader();
qh = new QuoteHistory();
trader.getAssistant().getQuoteHistories().put(REQUEST_ID, qh);
trader.addErrorListener(this);
}
public void run() {
try {
download();
if (!isCancelled) {
writer = new PrintWriter(new BufferedWriter(new FileWriter(fileName, false)));
writeDescriptior();
writeBars();
writer.close();
backDataDialog.setProgress(100, "Done");
backDataDialog.signalCompleted();
String msg = priceBars.size() + " bars have been downloaded successfully.";
eventlogger.write(msg, "Info", 1);
MessageDialog.showMessage(backDataDialog, msg);
}
} catch (Throwable t) {
eventlogger.write(t);
MessageDialog.showError(backDataDialog, t.getMessage());
} finally {
backDataDialog.signalCompleted();
qh.setIsHistRequestCompleted(true);
trader.setIsPendingHistRequest(false);
trader.removeErrorListener(this);
}
}
public void error(int id, int errorCode, String errorMsg) {
hasSessionExpired = (errorCode == 2107) && errorMsg.contains("HMDS data farm connection is inactive");
if (hasSessionExpired) {
synchronized (this) {
notifyAll();
}
}
firstBarReached = (errorCode == 162 && errorMsg.contains("HMDS query returned no data"));
if (firstBarReached) {
firstBarReached = true;
qh.setIsHistRequestCompleted(true);
synchronized (trader) {
trader.setIsPendingHistRequest(false);
trader.notifyAll();
}
return;
}
if (errorCode == 162 || errorCode == 200 || errorCode == 321) {
cancel();
String msg = "Could not complete back data download." + lineSep + "Cause: " + errorMsg;
MessageDialog.showError(backDataDialog, msg);
}
}
private void download() throws InterruptedException {
backDataDialog.setProgress(0, "Downloading:");
int onlyRTHPriceBars = rthOnly ? 1 : 0;
// "trades" are not reported for Forex, only "midpoint".
String infoType = contract.m_exchange.equalsIgnoreCase("IDEALPRO") ? "MIDPOINT" : "TRADES";
Calendar cal = (Calendar) lastDate.clone();
isCancelled = false;
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd HH:mm:ss");
String endTime = dateFormat.format(cal.getTime());
double progress = 0;
// length of the period and last date for progress tracking purposes
long totalMillis = cal.getTimeInMillis() - firstDate.getTimeInMillis();
long lastDateMillis = cal.getTimeInMillis();
while (cal.after(firstDate)) {
// Note the time of the last request so that the next request can
// be timed to avoid the "pacing violation" error from the historical
// data server.
long requestTime = System.currentTimeMillis();
// Count the number of historical requests made within the current
// HMDS session. The maximum of 60 requests can be made in the context
// of a given session.
requestCounter++;
// Make historical data request
trader.getAssistant().getHistoricalData(REQUEST_ID, contract, endTime, "5 D", barSize, infoType,
onlyRTHPriceBars, 2);
// Wait until the response is returned
synchronized (trader) {
while (!qh.getIsHistRequestCompleted()) {
// wait until the entire price bar history is returned
trader.wait();
}
}
// For certain contracts, the data is not available as far back as
// the beginning of the requested interval. If that's the case, simply
// bail out and use the data set that was received so far.
if (firstBarReached || isCancelled) {
return;
}
// Use the timestamp of the first bar in the received
// block of bars as the "end time" for the next historical data
// request.
long firstBarMillis = qh.getFirstPriceBar().getDate();
cal.setTimeInMillis(firstBarMillis);
endTime = dateFormat.format(cal.getTime());
// Add the just received block of bars to the cumulative set of
// bars and clear the current block for the next request
List allBars = qh.getAll();
priceBars.addAll(0, allBars);
allBars.clear();
qh.setIsHistRequestCompleted(false);
// IB's historical server uses a "throttle" mechanism to control
// the number of requests that can be made within a session. If we
// reach that number, we'll wait until the current HMDS session
// expires so that a new bulk of request can be made when a new
// HMDS session is created. Most of the time, when a session expires,
// a corresponding error 2107 is fired, so we know when we can resume
// making historical requests. If the error is NOT fired, we'll simply
// wait for MAX_WAITING_TIME to make sure that the session has expired.
if (requestCounter == MAX_REQUESTS_IN_HMDS_SESSION) {
eventlogger.write("Waiting for HMDS session to expire (in about 2 to 3 minutes)", "Info", 1);
backDataDialog.setProgress("Waiting for HMDS session to expire...");
long waitingTimeStart = System.currentTimeMillis();
long elapsedWaitingTime = 0;
synchronized (this) {
while (!hasSessionExpired && elapsedWaitingTime < MAX_WAITING_TIME) {
wait(MAX_WAITING_TIME - elapsedWaitingTime);
elapsedWaitingTime = System.currentTimeMillis() - waitingTimeStart;
}
}
//Thread.sleep(1000); // extra second to finalize the current HMDS session
// we are in the new HMDS session now, so reset the counter
requestCounter = 0;
backDataDialog.setProgress("Resuming download");
}
long now = System.currentTimeMillis();
long elapsedSinceLastRequest = now - requestTime;
long remainingTimeToSleep = MAX_FREQUENCY_MILLIS - elapsedSinceLastRequest;
if (remainingTimeToSleep > 0) {
// sleep to avoid "pacing violation"
Thread.sleep(remainingTimeToSleep);
}
// show download progress
progress = 100 * ( (lastDateMillis - firstBarMillis) / (double) totalMillis);
backDataDialog.setProgress( (int) progress, "Downloading:");
}
}
private void setPeriodBoundaries() {
lastDate = Calendar.getInstance();
if (contract.m_expiry != null) {
String expiration = contract.m_expiry;
int expirationYear = Integer.valueOf(expiration.substring(0, 4));
int expirationMonth = Integer.valueOf(expiration.substring(4, 6));
Calendar expirationDate = Calendar.getInstance();
expirationDate.set(Calendar.YEAR, expirationYear);
expirationDate.set(Calendar.MONTH, expirationMonth - 1);
expirationDate.set(Calendar.WEEK_OF_MONTH, 3);
expirationDate.set(Calendar.DAY_OF_WEEK, Calendar.FRIDAY);
if (lastDate.after(expirationDate)) {
contract.m_includeExpired = true;
lastDate = expirationDate;
}
}
// approximate start of the first trading day 1 year ago
int maxDays = 361;
firstDate = (Calendar) lastDate.clone();
firstDate.add(Calendar.DAY_OF_YEAR, -maxDays);
}
public void cancel() {
preferences.putInt(REQUEST_COUNTER, requestCounter);
isCancelled = true;
qh.setIsHistRequestCompleted(true);
synchronized (trader) {
trader.setIsPendingHistRequest(false);
trader.notifyAll();
}
trader.removeErrorListener(this);
}
private void writeDescriptior() {
writer.println("columns=7");
writer.println("dateColumn=1");
writer.println("timeColumn=2");
writer.println("openColumn=3");
writer.println("highColumn=4");
writer.println("lowColumn=5");
writer.println("closeColumn=6");
writer.println("volumeColumn=7");
writer.println("separator=,");
writer.println("dateFormat=MM/dd/yyyy");
writer.println("timeFormat=HH:mm");
writer.println("timeZone=EST");
writer.println();
}
private void writeBars() {
SimpleDateFormat dateFormat = new SimpleDateFormat("MM/dd/yyyy,HH:mm,");
double progress = 0;
long barsWritten = 0;
int size = priceBars.size();
for (PriceBar priceBar : priceBars) {
String dateTime = dateFormat.format(new Date(priceBar.getDate()));
String line = dateTime + priceBar.getOpen() + "," + priceBar.getHigh() + ",";
line += priceBar.getLow() + "," + priceBar.getClose();
line += "," + priceBar.getVolume();
writer.println(line);
barsWritten++;
if (barsWritten % 100 == 0) {
progress = (100 * barsWritten) / size;
backDataDialog.setProgress( (int) progress, "Writing to file:");
}
}
}
}