Merge pull request #808 from e2thenegpii/master

Support new TSP funds in 2020

Author: bashtage

docs/source/whatsnew/v0.9.0.txt

@@ -30,6 +30,7 @@ Backwards incompatible API changes
 - Removed Robinhood which no longer exists
 - Immediately deprecated AlphaVantage quote reader which used an end point that has been retired
 - Immediately deprecated Enigma which has substantially changed their business model and API
+- Immediately deprecated old symbol names for TSP to reflect the new website API
 
 .. _whatsnew_090.bug_fixes:
 
@@ -42,6 +43,7 @@ Bug Fixes
 - Correct NASDAQ symbols fields link (:issue:`715`)
 - Fix Yahoo! actions bug due to change in split format (:issue:`755`)
 - Fix FutureWarning from pandas import (:issue:`762`)
+- Update TSP web scraper to new site (:issue:`740`)
 
 Contributors
 ~~~~~~~~~~~~
@@ -56,4 +58,4 @@ Contributors
 - Gábor Lipták
 - Kevin Sheppard
 - David Stephens
-
+- Eldon Allred

pandas_datareader/tests/test_tsp.py

@@ -11,10 +11,19 @@ class TestTSPFunds(object):
     def test_get_allfunds(self):
         tspdata = tsp.TSPReader(start="2015-11-2", end="2015-11-2").read()
 
-        assert len(tspdata == 1)
+        assert len(tspdata) == 1
 
         assert round(tspdata["I Fund"][dt.datetime(2015, 11, 2)], 5) == 25.0058
 
+    def test_get_one_fund(self):
+        tspdata = tsp.TSPReader(
+            start="2015-11-2", end="2015-11-2", symbols=("I Fund",)
+        ).read()
+
+        assert len(tspdata) == 1
+
+        assert tspdata.columns.values.tolist() == ["I Fund"]
+
     def test_sanitize_response(self):
         class response(object):
             pass

pandas_datareader/tsp.py

@@ -27,9 +27,29 @@ class TSPReader(_BaseReader):
         requests.sessions.Session instance to be used
     """
 
+    all_symbols = frozenset(
+        (
+            "L Income",
+            "L 2025",
+            "L 2030",
+            "L 2035",
+            "L 2040",
+            "L 2045",
+            "L 2050",
+            "L 2055",
+            "L 2060",
+            "L 2065",
+            "G Fund",
+            "F Fund",
+            "C Fund",
+            "S Fund",
+            "I Fund",
+        )
+    )
+
     def __init__(
         self,
-        symbols=("Linc", "L2020", "L2030", "L2040", "L2050", "G", "F", "C", "S", "I"),
+        symbols=all_symbols,
         start=None,
         end=None,
         retry_count=3,
@@ -49,22 +69,24 @@ def __init__(
     @property
     def url(self):
         """API URL"""
-        return "https://www.tsp.gov/InvestmentFunds/FundPerformance/index.html"
+        return "https://secure.tsp.gov/components/CORS/getSharePricesRaw.html"
 
     def read(self):
         """ read one data from specified URL """
         df = super(TSPReader, self).read()
         df.columns = map(lambda x: x.strip(), df.columns)
+        df.drop(columns=self.all_symbols - set(self.symbols), inplace=True)
         return df
 
     @property
     def params(self):
         """Parameters to use in API calls"""
         return {
-            "startdate": self.start.strftime("%m/%d/%Y"),
-            "enddate": self.end.strftime("%m/%d/%Y"),
-            "fundgroup": self.symbols,
-            "whichButton": "CSV",
+            "startdate": self.start.strftime("%Y%m%d"),
+            "enddate": self.end.strftime("%Y%m%d"),
+            "download": "0",
+            "Lfunds": "1",
+            "InvFunds": "1",
         }
 
     @staticmethod