select_stock.py

# -*-coding=utf-8-*-
# 适用 tushare 0.7.5
__author__ = 'Rocky'
'''
http://30daydo.com
Contact: weigesysu@qq.com
'''
import tushare as ts
import pandas as pd
import os, datetime, time, Queue
from toolkit import Toolkit
from threading import Thread

q = Queue.Queue()

# 用来选股用的
pd.set_option('max_rows', None)

from configure.settings import get_engine
engine = get_engine('db_stock')
# 缺陷： 暂时不能保存为excel
class filter_stock():
    def __init__(self,retry=5,local=False):
        if local:
            for i in range(retry):
                try:
                    self.bases_save = ts.get_stock_basics()
                    # print(self.bases_save)
                    self.bases_save=self.bases_save.reset_index()
                    self.bases_save.to_csv('bases.csv')
                    self.bases_save.to_sql('bases',engine,if_exists='replace')
                    if self.bases_save:
                        break
                
                except Exception as e:
                    if i>=4:
                        self.bases_save=pd.DataFrame()
                        exit()                        
                    continue                  
        
        else:
            self.bases_save = pd.read_sql('bases',engine,index_col='index')
            self.base=self.bases_save

        # 因为网速问题，手动从本地抓取
        self.today = time.strftime("%Y-%m-%d", time.localtime())
        # self.base = pd.read_csv('bases.csv', dtype={'code': np.str})
        self.all_code = self.base['code'].values
        self.working_count = 0
        self.mystocklist = Toolkit.read_stock('mystock.csv')

    # 保存为excel 文件 这个时候csv 乱码,excel正常.
    def save_data_excel(self):
        df = ts.get_stock_basics()

        df.to_csv(self.today + '.csv', encoding='gbk')
        df_x = pd.read_csv(self.today + '.csv', encoding='gbk')
        df_x.to_excel(self.today + '.xls', encoding='gbk')
        os.remove(self.today + '.csv')

    def insert_garbe(self):
        print('*' * 30)
        print('\n')

    def showInfo(self, df):
        print('*' * 30)
        print('\n')
        print(df.info())
        print('*' * 30)
        print('\n')
        print(df.dtypes)
        self.insert_garbe()
        print(df.describe())

    # 计算每个地区有多少上市公司
    def count_area(self, writeable=False):
        count = self.base['area'].value_counts()
        print(count)
        print(type(count))
        if writeable:
            count.to_csv('各省的上市公司数目.csv')
        return count

    # 显示你要的某个省的上市公司
    def get_area(self, area, writeable=False):
        user_area = self.base[self.base['area'] == area]
        user_area.sort_values('timeToMarket', inplace=True, ascending=False)
        if writeable:
            filename = area + '.csv'
            user_area.to_csv(filename)
        return user_area

    # 获取所有地区的分类个股
    def get_all_location(self):
        series = self.count_area()
        index = series.index
        for i in index:
            name = unicode(i)
            self.get_area(name, writeable=True)

    # 找出指定日期后的次新股
    def fetch_new_ipo(self, start_time, writeable=False):
        # 需要继续转化为日期类型

        df = self.base.loc[self.base['timeToMarket'] > start_time]
        df.sort_values('timeToMarket', inplace=True, ascending=False)
        if writeable == True:
            df.to_csv("New_IPO.csv")
        # sum_a=df['pe'].sum()

        pe_av = df[df['pe'] != 0]['pe'].mean()
        pe_all_av = self.base[self.base['pe'] != 0]['pe'].mean()
        print(u"平均市盈率为 ", pe_av)
        print('A股的平均市盈率为 ', pe_all_av)
        return df

    # 获取成分股
    def get_chengfenggu(self, writeable=False):
        s50 = ts.get_sz50s()
        if writeable == True:
            s50.to_excel('sz50.xls')
        list_s50 = s50['code'].values.tolist()
        # print(type(s50))
        # print(type(list_s50))
        # 返回list类型
        return list_s50

    # 计算一个票从最高位到目前 下跌多少 计算跌幅
    def drop_down_from_high(self, start, code):

        end_day = datetime.date(datetime.date.today().year, datetime.date.today().month, datetime.date.today().day)
        end_day = end_day.strftime("%Y-%m-%d")
        # print(e)nd_day
        # print(start)
        total = ts.get_k_data(code=code, start=start, end=end_day)
        # print(total)
        high = total['high'].max()
        high_day = total.loc[total['high'] == high]['date'].values[0]

        print(high)
        print(high_day)
        current = total['close'].values[-1]
        print(current)
        percent = round((current - high) / high * 100, 2)
        print(percent)
        return percent

    def loop_each_cixin(self):
        df = self.fetch_new_ipo(20170101, writeable=False)
        all_code = df['code'].values
        print(all_code)
        # exit()
        percents = []
        for each in all_code:
            print(each)
            # print(type(each))
            percent = self.drop_down_from_high('2017-01-01', each)
            percents.append(percent)

        df['Drop_Down'] = percents

        # print(df)

        df.sort_values('Drop_Down', ascending=True, inplace=True)
        # print(df)
        df.to_csv(self.today + '_drop_Down_cixin.csv')

    # 获取所有的ma5>ma10
    def macd(self):
        # df=self.fetch_new_ipo(writeable=True)
        # all_code=df['code'].values
        # all_code=self.get_all_code()
        # print(all_code)
        result = []
        for each_code in self.all_code:
            print(each_code)
            try:
                df_x = ts.get_k_data(code=each_code, start='2017-03-01')
            # 只找最近一个月的，所以no item的是停牌。
            except:
                print("Can't get k_data")
                continue
            if len(df_x) < 11:
                # return
                print("no item")
                continue
            ma5 = df_x['close'][-5:].mean()
            ma10 = df_x['close'][-10:].mean()
            if ma5 > ma10:
                # print("m5>m10: ",each_code," ",self.base[self.base['code']==each_code]['name'].values[0], "ma5: ",ma5,' m10: ',ma10)
                temp = [each_code, self.base[self.base['code'] == each_code]['name'].values[0]]
                print(temp)
                result.append(temp)
        print(result)
        print("Done")
        return result

    # 返回所有股票的代码
    def get_all_code(self):
        return self.all_code

    # 获取成交量的ma5 或者10
    def volume_calculate(self, codes):
        delta_day = 180 * 7 / 5
        end_day = datetime.date(datetime.date.today().year, datetime.date.today().month, datetime.date.today().day)
        start_day = end_day - datetime.timedelta(delta_day)

        start_day = start_day.strftime("%Y-%m-%d")
        end_day = end_day.strftime("%Y-%m-%d")
        print(start_day)
        print(end_day)
        result_m5_large = []
        result_m5_small = []
        for each_code in codes:
            # print(e)ach_code
            try:
                df = ts.get_k_data(each_code, start=start_day, end=end_day)
                print(df)
            except Exception as e:
                print("Failed to get")
                print(e)
                continue

            if len(df) < 20:
                # print("not long enough")
                continue
            print(each_code)
            all_mean = df['volume'].mean()
            m5_volume_m = df['volume'][-5:].mean()
            m10_volume_m = df['volume'][-10:].mean()
            last_vol = df['volume'][-1]  # 这里会不会有问题？？？
            # 在这里分几个分支，放量 180天均量的4倍
            if m5_volume_m > (4.0 * all_mean):
                print("m5 > m_all_avg ")
                print(each_code,)
                temp = self.base[self.base['code'] == each_code]['name'].values[0]
                print(temp)
                result_m5_large.append(each_code)

            # 成交量萎缩
            if last_vol < (m5_volume_m / 3.0):
                result_m5_small.append(each_code)
        return result_m5_large, result_m5_large

    def turnover_check(self):
        delta_day = 60 * 7 / 5
        end_day = datetime.date(datetime.date.today().year, datetime.date.today().month, datetime.date.today().day)
        start_day = end_day - datetime.timedelta(delta_day)

        start_day = start_day.strftime("%Y-%m-%d")
        end_day = end_day.strftime("%Y-%m-%d")
        print(start_day)
        print(end_day)
        for each_code in self.all_code:
            try:
                df = ts.get_hist_data(code=each_code, start=start_day, end=end_day)
            except:
                print("Failed to get data")
                continue
            mv5 = df['v_ma5'][-1]
            mv20 = df['v_ma20'][-1]
            mv_all = df['volume'].mean()


    # 写入csv文件
    def write_to_text(self):
        print("On write")
        r = self.macd()
        filename = self.today + "-macd.csv"
        f = open(filename, 'w')
        for i in r:
            f.write(i[0])
            f.write(',')
            f.write(i[1])
            f.write('\n')
        f.close()

    def saveList(self, l, name):
        f = open(self.today + name + '.csv', 'w')
        if len(l) == 0:
            return False
        for i in l:
            f.write(i)
            f.write(',')
            name = self.base[self.base['code'] == i]['name'].values[0]
            f.write(name)
            f.write('\n')
        f.close()
        return True

    # 读取自己的csv文件
    def read_csv(self):
        filename = self.today + "-macd.csv"
        df = pd.read_csv(filename)
        print(df)

    # 持股从高点下跌幅度
    def own_drop_down(self):
        for i in self.mystocklist:
            print(i)
            self.drop_down_from_high(code=i, start='2017-01-01')
            print('\n')

    # 持股跌破均线
    def _break_line(self, codes, k_type):
        delta_day = 60 * 7 / 5
        end_day = datetime.date(datetime.date.today().year, datetime.date.today().month, datetime.date.today().day)
        start_day = end_day - datetime.timedelta(delta_day)

        start_day = start_day.strftime("%Y-%m-%d")
        end_day = end_day.strftime("%Y-%m-%d")
        print(start_day)
        print(end_day)
        all_break = []

        for i in codes:
            try:
                df = ts.get_hist_data(code=i, start=start_day, end=end_day)
                if len(df) == 0:
                    continue
            except Exception as e:
                print(e)
                continue
            else:
                self.working_count = self.working_count + 1
                current = df['close'][0]
                ma5 = df['ma5'][0]
                ma10 = df['ma10'][0]
                ma20 = df['ma20'][0]
                ma_dict = {'5': ma5, '10': ma10, '20': ma20}
                ma_x = ma_dict[k_type]
                # print(ma_x)
                if current < ma_x:
                    print('破位')
                    print(i, " current: ", current)
                    print(self.base[self.base['code'] == i]['name'].values[0], " ")
                    print("holding place: ", ma_x)
                    print("Break MA", k_type, "\n")
                    all_break.append(i)
        return all_break

    # 检查自己的持仓或者市场所有破位的
    def break_line(self, code, k_type='20', writeable=False, mystock=False):

        all_break = self._break_line(code, k_type)
        l = len(all_break)
        beaking_rate = l * 1.00 / self.working_count * 100
        print("how many break: ", l)
        print("break Line rate ", beaking_rate)
        if mystock == False:
            name = '_all_'
        else:
            name = '_my__'
        if writeable:
            f = open(self.today + name + 'break_line_' + k_type + '.csv', 'w')
            f.write("Breaking rate: %f\n\n" % beaking_rate)
            f.write('\n'.join(all_break))
            f.close()

    def _break_line_thread(self, codes, k_type='5'):
        delta_day = 60 * 7 / 5
        end_day = datetime.date(datetime.date.today().year, datetime.date.today().month, datetime.date.today().day)
        start_day = end_day - datetime.timedelta(delta_day)

        start_day = start_day.strftime("%Y-%m-%d")
        end_day = end_day.strftime("%Y-%m-%d")
        print(start_day)
        print(end_day)
        all_break = []
        for i in codes:
            try:
                df = ts.get_hist_data(code=i, start=start_day, end=end_day)
                if len(df) == 0:
                    continue
            except Exception as e:
                print(e)
                continue
            else:
                self.working_count = self.working_count + 1
                current = df['close'][0]
                ma5 = df['ma5'][0]
                ma10 = df['ma10'][0]
                ma20 = df['ma20'][0]
                ma_dict = {'5': ma5, '10': ma10, '20': ma20}
                ma_x = ma_dict[k_type]
                # print(ma_x)
                if current > ma_x:
                    print(i, " current: ", current)
                    print(self.base[self.base['code'] == i]['name'].values[0], " ")

                    print("Break MA", k_type, "\n")
                    all_break.append(i)
        q.put(all_break)

    def multi_thread_break_line(self, ktype='20'):
        total = len(self.all_code)
        thread_num = 10
        delta = total / thread_num
        delta_left = total % thread_num
        t = []
        i = 0
        for i in range(thread_num):
            sub_code = self.all_code[i * delta:(i + 1) * delta]
            t_temp = Thread(target=self._break_line_thread, args=(sub_code, ktype))
            t.append(t_temp)
        if delta_left != 0:
            sub_code = self.all_code[i * delta:i * delta + delta_left]
            t_temp = Thread(target=self._break_line_thread, args=(sub_code, ktype))
            t.append(t_temp)

        for i in range(len(t)):
            t[i].start()

        for j in range(len(t)):
            t[j].join()
        result = []
        print("working done")
        while not q.empty():
            result.append(q.get())
        ff = open(self.today + '_high_m%s.csv' % ktype, 'w')
        for kk in result:
            print(kk)
            for k in kk:
                ff.write(k)
                ff.write(',')
                ff.write(self.base[self.base['code'] == k]['name'].values[0])
                ff.write('\n')

        ff.close()

        # 计算大盘的相关系，看关系如何

    def relation(self):
        sh_index = ts.get_k_data('000001', index=True, start='2012-01-01')
        sh = sh_index['close'].values
        print(sh)
        vol_close = sh_index.corr()
        print(vol_close)
        '''
        sz_index=ts.get_k_data('399001',index=True)
        sz=sz_index['close'].values
        print(sz)

        cy_index=ts.get_k_data('399006',index=True)
        s1=Series(sh)
        s2=Series(sz)
        print(s1.corr(s2))
        '''

    # 寻找业绩两年未负的，以防要st
    def profit(self):
        df_2016 = ts.get_report_data(2016, 4)

        # 第四季度就是年报
        # df= df.sort_values('profits_yoy',ascending=False)
        # df.to_excel('profit.xls')
        df_2015 = ts.get_report_data(2015, 4)
        df_2016.to_excel('2016_report.xls')
        df_2015.to_excel('2015_report.xls')
        code_2015_lost = df_2015[df_2015['net_profits'] < 0]['code'].values
        code_2016_lost = df_2016[df_2016['net_profits'] < 0]['code'].values

        print(code_2015_lost)
        print(code_2016_lost)
        two_year_lost = []
        # two_year_lost_name=[]
        for i in code_2015_lost:
            if i in code_2016_lost:
                print(i,)
                # name=self.base[self.base['code']==i].values[0]
                two_year_lost.append(i)

        self.saveList(two_year_lost, 'st_dangours.csv')

        # df_2014=ts.get_report_data(2014,4)

    def mydaily_check(self):
        self.break_line(self.mystocklist, k_type='5', writeable=True, mystock=True)

    def all_stock(self):
        self.multi_thread_break_line('20')

#破净资产的股票
def get_break_bvps():
    base_info = ts.get_stock_basics()
    current_prices = ts.get_today_all()


    current_prices[current_prices['code'] == '000625']['trade'].values[0]
    base_info.loc['000625']['bvps']

def main():
    folder = os.path.join(os.path.dirname(__file__), 'data')
    if os.path.exists(folder) == False:
        os.mkdir(folder)
    os.chdir(folder)

    obj = filter_stock(local=True)
    # 留下来的函数都是有用的
    # obj.count_area(writeable=True)
    # df=obj.get_area('广东',writeable=True)
    # obj.showInfo(df)
    # df=obj.get_area('深圳',writeable=True)
    # obj.showInfo(df)
    # obj.get_all_location()
    # obj.fetch_new_ipo(20170101,writeable=False)

    # obj.drop_down_from_high('2017-01-01','300580')
    # obj.loop_each_cixin()

    # df=obj.get_all_code()
    # result=obj.volume_calculate(df)
    # obj.saveList(result)

    # df=obj.get_chengfenggu()
    # large,small=obj.volume_calculate(df)
    # obj.saveList(large,'large')
    # obj.saveList(small,'small')
    # obj.write_to_text()
    # obj.read_csv()
    # obj.own_drop_down()
    # obj.volume_calculate()
    # obj.break_line()
    # obj.save_data_excel()
    # obj.break_line(mine=False,k_type='5')
    # obj.multi_thread()
    # code=obj.get_chengfenggu()
    # obj.break_line(code)
    # obj.big_deal('603918',400,'2017-04-22')
    # obj.current_day_ticks()
    # obj.relation()
    # obj.profit()

    # obj.mydaily_check()
    # obj.all_stock()


if __name__ == "__main__":
    start_time = datetime.datetime.now()
    print(start_time)
    main()
    end_time = datetime.datetime.now()
    print(end_time)
    print("time use : ", (end_time - start_time).seconds)