[py] qbquant stock list, creat stock and index list

README.org

@@ -95,9 +95,8 @@ abquant 分为三大模块 a) 数据获取 b) 策略回测 c) 模拟/实盘交
        # 但是不包含除权数据，除权数据不是经常更新，请手动 "abquant stock xdxr"
        abquant save base
 
-       # 股票除权除息
-       abquant stock xdxr
-       abquant stock xdxr --add '["000001", "300001", "600001"]'
+       # 股票和指数列表获取
+       abquant stock list
 
        # 保存所有股票日线，日线指数
        abquant stock day
@@ -120,6 +119,10 @@ abquant 分为三大模块 a) 数据获取 b) 策略回测 c) 模拟/实盘交
        abquant stock min -f '["1min", "5min"]'
        abquant stock min -f '["1min", "5min"]' --add '["000001", "300001", "600001"]'
 
+       # 股票除权除息
+       abquant stock xdxr
+       abquant stock xdxr --add '["000001", "300001", "600001"]'
+
        # 股票数据板块
        abquant stock block
 

abquant/cli.py

@@ -6,6 +6,7 @@
 from abquant.utils.logger import set_loggers
 from abquant.utils.logger import system_log as slog
 from abquant.data.base import (
+    create_stock_list,
     create_stock_day,
     create_stock_min,
     create_stock_xdxr,
@@ -54,6 +55,12 @@ def etf():
     """Manages etf."""
 
 
+@stock.command("list")
+def stock_list():
+    """stock list."""
+    create_stock_list()
+
+
 @stock.command("day")
 @click.option(
     "ow",

abquant/data/base.py

@@ -21,6 +21,10 @@ def getClassName(self):
 
 
 class ISecurityVisitor(object, metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def create_list(self, iSecurity: ISecurity):
+        pass
+
     @abc.abstractmethod
     def create_day(self, iSecurity: ISecurity):
         pass
@@ -39,17 +43,23 @@ def __init__(self, *args, **kwargs):
         "docstring"
         pass
 
+    def create_list(self, iSecurity: ISecurity):
+        if getattr(iSecurity, "create_list"):
+            iSecurity.create_list(ins_type=INSTRUMENT_TYPE.INDX)
+            iSecurity.create_list(ins_type=INSTRUMENT_TYPE.ETF)
+            iSecurity.create_list(ins_type=INSTRUMENT_TYPE.CS)
+
     def create_day(self, iSecurity: ISecurity):
         if getattr(iSecurity, "create_day"):
             iSecurity.create_day(ins_type=INSTRUMENT_TYPE.INDX)
             iSecurity.create_day(ins_type=INSTRUMENT_TYPE.ETF)
-            iSecurity.create_day()
+            iSecurity.create_day(ins_type=INSTRUMENT_TYPE.CS)
 
     def create_min(self, iSecurity: ISecurity):
         if getattr(iSecurity, "create_min"):
             iSecurity.create_min(ins_type=INSTRUMENT_TYPE.INDX)
             iSecurity.create_min(ins_type=INSTRUMENT_TYPE.ETF)
-            iSecurity.create_min()
+            iSecurity.create_min(ins_type=INSTRUMENT_TYPE.CS)
 
     def create_xdxr(self, iSecurity: ISecurity):
         if getattr(iSecurity, "create_xdxr", None):
@@ -99,6 +109,15 @@ def regist_securities(
     regist_securities(securities, SecurityVisitor())
 
 
+@time_counter
+def create_stock_list():
+    broker = get_broker("tdx")
+    brokers_api = [get_broker_api(b) for b in ["tdx", "ths", "qa"]]
+    e = broker.Stock()
+    e.create_list(brokers_api=brokers_api, ins_type=INSTRUMENT_TYPE.INDX)
+    e.create_list(brokers_api=brokers_api, ins_type=INSTRUMENT_TYPE.CS)
+
+
 @time_counter
 def create_stock_day(codes):
     broker = get_broker()

abquant/data/tdx.py

@@ -40,6 +40,47 @@ def accept(self, visitor: ISecurityVisitor) -> None:
         visitor.create_min(self)
         visitor.create_xdxr(self)
 
+    def create_list(self, *args, **kwargs):
+        """save security list
+
+        Keyword Arguments:
+            client {[type]} -- [description] (default: {DATABASE})
+        """
+        brokers_api = kwargs.get("brokers_api", [])
+        self._db.drop_collection("stock_list")
+        ins_type = kwargs.pop("ins_type", INSTRUMENT_TYPE.CS)
+        if ins_type == INSTRUMENT_TYPE.INDX:
+            self._db.drop_collection("index_list")
+            coll = self._db.index_list
+        else:
+            self._db.drop_collection("stock_list")
+            coll = self._db.stock_list
+        coll.create_index([("code", pymongo.ASCENDING)])
+
+        def saving_work(b_api):
+            try:
+                begin = datetime.datetime.now()
+                df = b_api.get_stock_list(type_=ins_type)
+                if isinstance(df, pd.DataFrame) and df.empty:
+                    slog.warn("df is empty.")
+                    return
+                if not isinstance(df, pd.DataFrame) and not df:
+                    return
+                coll.insert_many(to_json_from_pandas(df))
+                finish = datetime.datetime.now()
+                interval = (finish - begin).total_seconds()
+                text = "{} list {}, {:<04.2}s".format(
+                    ins_type, b_api.my_name(), interval
+                )
+                for _ in trange(df.size, desc=text):
+                    pass
+            except Exception as e:
+                slog.error("{}".format(e))
+
+        tqdm.set_lock(RLock())
+        with ThreadPoolExecutor(max_workers=1) as p:
+            p.map(partial(saving_work), brokers_api)
+
     def create_day(self, *args, **kwargs):
         """save INSTRUMENT_TYPE.CS or INSTRUMENT_TYPE.INDX day
 
@@ -62,7 +103,10 @@ def create_day(self, *args, **kwargs):
             else self._db.stock_day
         )
         coll.create_index(
-            [("code", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING),]
+            [
+                ("code", pymongo.ASCENDING),
+                ("date_stamp", pymongo.ASCENDING),
+            ]
         )
         err = []
 
@@ -221,7 +265,8 @@ def create_xdxr(self, *args, **kwargs):
             [("code", pymongo.ASCENDING), ("date", pymongo.ASCENDING)], unique=True
         )
         coll_adj.create_index(
-            [("code", pymongo.ASCENDING), ("date", pymongo.ASCENDING)], unique=True,
+            [("code", pymongo.ASCENDING), ("date", pymongo.ASCENDING)],
+            unique=True,
         )
         err = []
 
@@ -256,7 +301,8 @@ def saving_work(idx):
                     finish = datetime.datetime.now()
                     interval = (finish - begin).total_seconds()
                     text = "{}, {:<04.2}s".format(
-                        "{} {}".format(ins_type, code), interval,
+                        "{} {}".format(ins_type, code),
+                        interval,
                     )
                     for _ in trange(data.size, desc=text):
                         pass
@@ -311,7 +357,10 @@ def saving_work(idx):
                 coll.insert_many(to_json_from_pandas(df))
                 finish = datetime.datetime.now()
                 interval = (finish - begin).total_seconds()
-                text = "{}, {:<04.2}s".format("{} {}".format(ins_type, code), interval,)
+                text = "{}, {:<04.2}s".format(
+                    "{} {}".format(ins_type, code),
+                    interval,
+                )
                 for _ in trange(df.size, desc=text):
                     pass
             except Exception as e:
@@ -440,9 +489,43 @@ def __init__(self, *args, **kwargs):
         self.freqs = kwargs.get("freqs", ["1min", "5min", "15min", "30min", "60min"])
 
     def accept(self, visitor: ISecurityVisitor) -> None:
+        visitor.create_list(self)
         visitor.create_day(self)
         visitor.create_min(self)
 
+    def create_list(self, *args, **kwargs):
+        """save etf list
+
+        Keyword Arguments:
+            client {[type]} -- [description] (default: {DATABASE})
+        """
+        brokers_api = kwargs.get("brokers_api", [])
+        self._db.drop_collection("etf_list")
+        coll = self._db.etf_list
+        coll.create_index([("code", pymongo.ASCENDING)])
+
+        def saving_work(b_api):
+            try:
+                begin = datetime.datetime.now()
+                df = b_api.get_stock_list(type_=INSTRUMENT_TYPE.ETF)
+                if isinstance(df, pd.DataFrame) and df.empty:
+                    slog.warn("df is empty.")
+                    return
+                if not isinstance(df, pd.DataFrame) and not df:
+                    return
+                coll.insert_many(to_json_from_pandas(df))
+                finish = datetime.datetime.now()
+                interval = (finish - begin).total_seconds()
+                text = "etf list {}, {:<04.2}s".format(b_api.my_name(), interval)
+                for _ in trange(df.size, desc=text):
+                    pass
+            except Exception as e:
+                slog.error("{}".format(e))
+
+        tqdm.set_lock(RLock())
+        with ThreadPoolExecutor(max_workers=1) as p:
+            p.map(partial(saving_work), brokers_api)
+
     def create_day(self, *args, **kwargs):
         """save etf_day
 
@@ -465,7 +548,10 @@ def create_day(self, *args, **kwargs):
             else self._db.stock_day
         )
         coll.create_index(
-            [("code", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING),]
+            [
+                ("code", pymongo.ASCENDING),
+                ("date_stamp", pymongo.ASCENDING),
+            ]
         )
         err = []
 
@@ -601,36 +687,3 @@ def saving_work(idx):
             p.map(partial(saving_work), list(range(len(instruments))))
 
         save_error_log(err, "{}_{}_min".format(self.getClassName(), ins_type))
-
-    def create_list(self, *args, **kwargs):
-        """save etf list
-
-        Keyword Arguments:
-            client {[type]} -- [description] (default: {DATABASE})
-        """
-        brokers_api = kwargs.get("brokers_api", [])
-        self._db.drop_collection("etf_list")
-        coll = self._db.etf_list
-        coll.create_index([("code", pymongo.ASCENDING)])
-
-        def saving_work(b_api):
-            try:
-                begin = datetime.datetime.now()
-                df = b_api.get_stock_list(type_=INSTRUMENT_TYPE.ETF)
-                if isinstance(df, pd.DataFrame) and df.empty:
-                    slog.warn("df is empty.")
-                    return
-                if not isinstance(df, pd.DataFrame) and not df:
-                    return
-                coll.insert_many(to_json_from_pandas(df))
-                finish = datetime.datetime.now()
-                interval = (finish - begin).total_seconds()
-                text = "etf list {}, {:<04.2}s".format(b_api.my_name(), interval)
-                for _ in trange(df.size, desc=text):
-                    pass
-            except Exception as e:
-                slog.error("{}".format(e))
-
-        tqdm.set_lock(RLock())
-        with ThreadPoolExecutor(max_workers=1) as p:
-            p.map(partial(saving_work), brokers_api)

binding/test/test_stockmin.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 
 import unittest
-from pyabqstockmin import PyStockMin as stockmin
 from pyabquant import FQ_TYPE, PyAbquant
+from pyabqstockmin import PyStockMin as stockmin
 import pandas as pd
 
 pd.set_option("display.max_rows", None)

pyproject.toml

@@ -26,6 +26,7 @@ pyarrow = "^4.0.0"
 conan = "^1.34.0"
 black = "^21.5b1"
 pytest = "^6.2.4"
+pudb = "^2021.1"
 
 [tool.poetry.scripts]
 abquant = "abquant.cli:cli"