Open source quantitative framework backtrader FAQ: develop sqlite data feed

See the full technical tutorial here

Many people have established their own local market database, hoping to send data from the local database to the backtrader for strategic use. A common method is to read the market data of the database into a pandas dataframe, and then pass the data of this data frame to the pandas feed data object of the backtrader, so that the strategy can be used.

However, some students don't want to transfer through pandas dataframe, but want to feed data directly from the database to the data feed object of the backtrader, which requires the development of a special data feed class for the database.

In the previous article, we introduced how to develop a data feed for the MySQL database, and this article introduces the development of a data feed for the sqlite database.

The following is a feed class SQLiteData provided by the backtrader community to read data from the sqlite database. You can give it a try.

import datetime as dt
from backtrader import TimeFrame
from backtrader.feed import DataBase
from backtrader import date2num


class SQLiteData(DataBase):
    '''
    Fetches data from SQLite, and wraps it into a Feed consumable by cerebro
    takes url connection string in form of :
        sqlite://{database}

    this implementation assumes a single table (historical_data) with all prices,
    conforming to a schema similar to the following:

            symbol        TEXT,
            date          TEXT (YYYY-mm-dd HH:mm),
            open          REAL,
            high          REAL,
            low           REAL,
            close         REAL,
            volume        INTEGER,
            unique (symbol, date)

    if your databases are set up differently, you can override the
    start() method.
    '''

    params = (
        ('database', None),
        ('symbol', 'XBTUSD'),
        ('tick_value', 0.01),
        ('timeframe ', TimeFrame.Minutes),
        ('compression', 1),
        ('fromdate', dt.datetime(1900, 1, 1)),
        ('todate', dt.datetime.max),

        # parameterized column indices for ease of overwriting/re-implementing
        ('datetime', 0),
        ('open', 1),
        ('high', 2),
        ('low', 3),
        ('close', 4),
        ('volume', 5),
        ('openinterest', -1),
    )

    def __init__(self):
        self._timeframe = self.p.timeframe
        self._compression = self.p.compression
        self._dataname = '{0}-{1:.2f}'.format(self.p.symbol, self.p.tick_value)

    def start(self):
        super(SQLiteData, self).start()
        self.biter = None
        self.preloaded = False

    def _preload(self):
        engine = self._connect_db()

        sql_query = "SELECT `date`,`open`,`high`,`low`,`close`,`volume` FROM `historical_data` WHERE `symbol` = '" + self.p.symbol + "' AND `date` between '" + self.p.fromdate.strftime(
            "%Y-%m-%d %H:%M:%S") + "' and '" + self.p.todate.strftime("%Y-%m-%d %H:%M:%S") + "' ORDER BY `date` ASC"

        result = engine.execute(sql_query)
        dbars = result.fetchall()
        result.close()

        self.biter = iter(dbars)

    def preload(self):
        if not self.biter:
            self._preload()

        while self.load():
            pass

        self._last()
        self.home()

        self.biter = None
        self.preloaded = True

    def _load(self):
        if self.preloaded:
            return False

        if not self.biter:
            self._preload()

        try:
            bar = next(self.biter)
        except StopIteration:
            return False

        for field in self.getlinealiases():
            if field == 'datetime':
                self.lines.datetime[0] = date2num(dt.datetime.strptime(bar[self.p.datetime], '%Y-%m-%d %H:%M:%S'))
            elif field == 'volume':
                self.lines.volume[0] = bar[self.p.volume]
            else:
                # get the column index
                colidx = getattr(self.params, field)
                if colidx < 0:
                # column not present -- skip
                    continue

                # get the line to be set
                line = getattr(self.lines, field)
                line[0] = float(bar[colidx])

        return True


    def _connect_db(self):
        from sqlalchemy import create_engine
        url = 'sqlite:///{0}'.format(self.p.database)
        engine = create_engine(url, echo=False)
        return engine

 

Posted 3 hours ago