feat(feed): arbitrary parsed objects (pandas, dicts) feed support

.travis.yml

@@ -1,7 +1,9 @@
 language: python
 python:
   - '3.7'
-
+cache:
+  - /usr/lib
+
 env:
   global:
     - secure: "SJPwuOKJDiGgOB+VyhJ/i/3Hb8z0leDo7hdgTp3X9+9dAdPQlwXSnCKoOEjLqpCz9gkr0pw6PCEnyDMcZo9ig2IBKt2y1r2M1Oc6c1d201EI+Fe6RGsRyG1StqU9iNSPZ9UEK+ld4Oa1FspNlFy1msd3N6sgpRRAa7HTENsCMKTl7Sq+wPwGtmm6qACCZN2Eqtth1ZyvwW5+64xRJPBNLcxNMnU2IJCPdn4j7MFsVpTufWAcV3kghvue0y73pKQRKnLrNGANt33uocU/PaKLYpGa/D0loEbRglgmG1ufnPCs4XyvRBgVN2iK2hLVHeXQvOXkfaH4joQzf+6PG/2UzP9EG/hRSSsI+Ul11mxxXrljwNm3a6N3E/JR7xhXWn1bGckLyoMm9tButedjs9zKhCRQWgh6f63l0JugFG86FLdAL1OpNjP77kpT+xwlrYaXvteJvr+TYu2H6QvsUJnT0QuRu9ydQD5UbaelVVZk4M8XK0cmD33HKEpBXNBAQjrvVcQu5VGJeyWgMYMmrDwjP4UsframpU8kAQa+cFu/ZaK624onROoxzD8/Am5sLdesJM6LWj1iSJSfHlGJT/KcfW+wM6+3JPuyR0X9Foyv1EkxzzqMMyOUZ04/hN8X681qBhEk9rnpPNphNnYOrmzS1JNXTNXt2/Boea5B58JNVDU="
@@ -10,10 +12,13 @@ sudo: required
 
 before_install:
   - sudo apt-get update
-  - sudo apt-get install build-essential wget
-  - wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz
-  - tar -xvzf ta-lib-0.4.0-src.tar.gz
-  - cd ta-lib/ && ./configure --prefix=/usr LDFLAGS="-lm" && make && sudo make install && cd .. && rm -r ta-lib
+  - set -e
+  - if [ ! -f /usr/lib/libta_lib.so ]; then
+          sudo apt-get install build-essential wget;
+          wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz;
+          tar -xvzf ta-lib-0.4.0-src.tar.gz;
+          cd ta-lib/ && ./configure --prefix=/usr LDFLAGS="-lm" && make && sudo make install && cd .. && rm -r ta-lib;
+    fi
   - python -m pip install poetry
   - poetry install
 
@@ -22,5 +27,5 @@ script:
   - export PYTHONPATH=.
   - poetry run tox -v
 
-after_success:
-  - poetry run coveralls
+# after_success:
+#   - poetry run coveralls

pyproject.toml

@@ -26,6 +26,7 @@ coveralls = "^1.9"
 pytest = "^5.3"
 tox = "^3.14"
 pytest-cov = "^2.8"
+pandas = "^0.25.3"
 
 [tool.poetry.extras]
 TALib = ["cython", "TA-Lib"]

quantworks/barfeed/csvfeed.py

@@ -158,8 +158,8 @@ def __init__(self, columnNames, dateTimeFormat, dailyBarTime, frequency, timezon
         self.__adjCloseColName = columnNames["adj_close"]
         self.__columnNames = columnNames
 
-    def _parseDate(self, dateString):
-        ret = datetime.datetime.strptime(dateString, self.__dateTimeFormat)
+    def _parseDate(self, dateTime):
+        ret = datetime.datetime.strptime(dateTime, self.__dateTimeFormat)
 
         if self.__dailyBarTime is not None:
             ret = datetime.datetime.combine(ret, self.__dailyBarTime)

quantworks/barfeed/listfeed.py

@@ -0,0 +1,193 @@
+# QuantWorks
+#
+# Copyright 2019 Tyler M Kontra
+# Copyright 2011-2015 Gabriel Martin Becedillas Ruiz
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+.. moduleauthor:: Alex McFarlane <alexander.mcfarlane@physics.org>, Tyler Kontra <tyler@tylerkontra.com>
+"""
+
+from quantworks.utils import dt
+from quantworks.barfeed import membf
+from quantworks.barfeed import csvfeed
+from quantworks import bar
+
+import datetime
+
+
+class BarFeed(membf.BarFeed):
+    """Base class for Iterable[Dict] based :class:`quantworks.barfeed.BarFeed`.
+
+    .. note::
+        This is a base class and should not be used directly.
+    """
+
+    def __init__(self, frequency, maxLen=None):
+        super(BarFeed, self).__init__(frequency, maxLen)
+
+        self.__barFilter = None
+        self.__dailyTime = datetime.time(0, 0, 0)
+
+    def getDailyBarTime(self):
+        return self.__dailyTime
+
+    def setDailyBarTime(self, time):
+        self.__dailyTime = time
+
+    def getBarFilter(self):
+        return self.__barFilter
+
+    def setBarFilter(self, barFilter):
+        self.__barFilter = barFilter
+
+    def _addBarsFromListofDicts(self, instrument, loadedBars):
+        loadedBars = filter(
+            lambda bar_: (bar_ is not None) and
+                (self.__barFilter is None or self.__barFilter.includeBar(bar_)),
+            loadedBars
+        )
+        self.addBarsFromSequence(instrument, loadedBars)
+
+
+class Feed(BarFeed):
+    """A BarFeed that loads bars from a custom feed that has the following columns:
+    ::
+
+                  Date Time     Open    Close     High      Low    Volume  Adj Close
+        2015-08-14 09:06:00  0.00690  0.00690  0.00690  0.00690  1.346117       9567
+
+
+    :param frequency: The frequency of the bars. Check :class:`quantworks.bar.Frequency`.
+    :param timezone: The default timezone to use to localize bars. Check :mod:`quantworks.marketsession`.
+    :type timezone: A pytz timezone.
+    :param maxLen: The maximum number of values that the :class:`quantworks.dataseries.bards.BarDataSeries` will hold.
+        Once a bounded length is full, when new items are added, a corresponding number of items are discarded from the
+        opposite end. If None then dataseries.DEFAULT_MAX_LEN is used.
+    :type maxLen: int.
+
+    .. note::
+        * The data should be sampled across regular time points, you can
+            regularlise (e.g. for 5min intervals) as::
+
+                df = df.set_index('Date Time').resample('s').interpolate().resample('5T').asfreq()
+                df = df.dropna().reset_index()
+            which is described in a SO [post](https://stackoverflow.com/a/39730730/4013571)
+        * It is ok if the **Adj Close** column is empty.
+        * When working with multiple instruments:
+
+         * If all the instruments loaded are in the same timezone, then the timezone parameter may not be specified.
+         * If any of the instruments loaded are in different timezones, then the timezone parameter should be set.
+    """
+
+    def __init__(self, frequency, timezone=None, maxLen=None):
+        super(Feed, self).__init__(frequency, maxLen)
+
+        self.__timezone = timezone
+        # Assume bars don't have adjusted close. This will be set to True after
+        # loading the first file if the adj_close column is there.
+        self.__haveAdjClose = False
+
+        self.__barClass = bar.BasicBar
+
+        self.__dateTimeFormat = "%Y-%m-%d %H:%M:%S"
+        self.__columnNames = {
+            "datetime": "Date Time",
+            "open": "Open",
+            "high": "High",
+            "low": "Low",
+            "close": "Close",
+            "volume": "Volume",
+            "adj_close": "Adj Close",
+        }
+        # self.__dateTimeFormat expects time to be set so there is no need to
+        # fix time.
+        self.setDailyBarTime(None)
+
+    def barsHaveAdjClose(self):
+        return self.__haveAdjClose
+
+    def setNoAdjClose(self):
+        self.__columnNames["adj_close"] = None
+        self.__haveAdjClose = False
+
+    def setColumnName(self, col, name):
+        self.__columnNames[col] = name
+
+    def setDateTimeFormat(self, dateTimeFormat):
+        self.__dateTimeFormat = dateTimeFormat
+
+    def setBarClass(self, barClass):
+        self.__barClass = barClass
+
+    def __localize_dt(self, dateTime):
+        # Localize the datetime if a timezone was given.
+        if self.__timezone:
+            return dt.localize(dateTime, self.__timezone)
+        else:
+            return dateTime
+
+    def addBarsFromListofDicts(self, instrument, list_of_dicts, timezone=None):
+        """Loads bars for a given instrument from a list of dictionaries.
+        The instrument gets registered in the bar feed.
+
+        :param instrument: Instrument identifier.
+        :type instrument: string.
+        :param list_of_dicts: A list of dicts. First item should contain
+            columns.
+        :type list_of_dicts: list
+        :param timezone: The timezone to use to localize bars. Check :mod:`quantworks.marketsession`.
+        :type timezone: A pytz timezone.
+        """
+
+        if timezone is None:
+            timezone = self.__timezone
+
+        if not isinstance(list_of_dicts, (list, tuple)):
+            raise ValueError('This function only supports types: {list, tuple}')
+        if not isinstance(list_of_dicts[0], dict):
+            raise ValueError('List should only contain dicts')
+
+        dicts_have_adj_close = self.__columnNames['adj_close'] in list_of_dicts[0].keys()
+
+        # Convert dicts to Bar(s)
+        loadedBars = map(
+          lambda row: bar.BasicBar(
+            self.__localize_dt(row[self.__columnNames['datetime']]),
+            row[self.__columnNames['open']],
+            row[self.__columnNames['high']],
+            row[self.__columnNames['low']],
+            row[self.__columnNames['close']],
+            row[self.__columnNames['volume']],
+            row[self.__columnNames['adj_close']],
+            self.getFrequency(),
+            extra = {key: row[key] for key in set(row.keys()).difference(self.__columnNames.values())}
+          ),
+          list_of_dicts
+        )
+
+        missing_columns = [
+            col for col in self.__columnNames.values()
+            if col not in list_of_dicts[0].keys()
+        ]
+        if missing_columns:
+            raise ValueError('Missing required columns: {}'.format(repr(missing_columns)))
+
+        super(Feed, self)._addBarsFromListofDicts(
+            instrument, loadedBars)
+
+        if dicts_have_adj_close:
+            self.__haveAdjClose = True
+        elif self.__haveAdjClose:
+            raise Exception("Previous bars had adjusted close and these ones don't have.")