layout/tools/t7.py

import copy
import os
import time
from collections import defaultdict
from datetime import datetime

import pandas as pd
from deap import base, creator, tools, algorithms

from tools.common import basedir, log


# 定义一个格式化函数
def format_date(date):
    return date.strftime('%Y-%m-%d')


def count_barcode_radio(data):
    df = pd.DataFrame(data)
    ratio_sites = dict()
    is_not_balance_list = []
    if df.empty:
        return ratio_sites, is_not_balance_list

    df['barcode'] = df['barcode'].str.slice(0, 16)
    barcode_df = pd.DataFrame(df['barcode'].str.split('', expand=True).iloc[:, 1:-1].values,
                              columns=['T' + str(x) for x in range(16)]).join(df['data_needed'])
    total = barcode_df['data_needed'].sum()

    for i in range(16):
        column = 'T' + str(i)
        col_df = barcode_df.groupby(column).agg({'data_needed': 'sum'})
        # 去掉N计数
        if 'N' in col_df.index:
            base_n_size = col_df.loc['N', 'data_needed']
            col_df = col_df.drop('N')
        else:
            base_n_size = 0
        col_df['ratio'] = (col_df['data_needed']) / (total - base_n_size)
        ratio = col_df['ratio'].to_dict()
        ratio_sites[i] = ratio
        A, B, C, D, E, F = list(), list(), list(), list(), list(), list()
        for decbase in ['A', 'T', 'C', 'G']:
            if decbase not in ratio:
                ratio[decbase] = 0
            if ratio[decbase] >= 0.6:
                A.append(decbase)
            if 0.2 <= ratio[decbase] < 0.6:
                B.append(decbase)
            if 0.15 <= ratio[decbase] < 0.2:
                C.append(decbase)
            if 0.1 <= ratio[decbase] < 0.15:
                D.append(decbase)
            if 0.08 <= ratio[decbase] < 0.1:
                E.append(decbase)
            if ratio[decbase] < 0.08:
                F.append(decbase)

        A_num, B_num, C_num, D_num, E_num, F_num = len(A), len(B), len(C), len(D), len(E), len(F)
        if not ((B_num + C_num + D_num == 4) or (F_num == 1 and (A_num + B_num) == 3) or (
                E_num == 1 and D_num == 1 and (A_num + B_num + C_num) == 2) or (
                        E_num == 1 and (A_num + B_num + C_num) == 3)):
            is_not_balance_list.append(
                '第%s位置，算出结果为 %s' % (i, ratio)
            )
    return ratio_sites, is_not_balance_list


# 定义遗传算法
class Ga:
    """
    # 定义遗传算法
    """

    def __init__(self, sheets):
        self.sheets = sheets

    # 定义个体的生成方式
    def generate_individual(self):
        individual = copy.deepcopy(self.sheets)  # 初始解作为个体
        return [individual]

    # 定义评估函数
    @staticmethod
    def evaluate(individual):
        total_data_needed_sum = 0
        xchip = 0
        try:
            for sheetname, data in individual[0][0].items():
                library_data = pd.DataFrame(data)

                size = library_data['data_needed'].sum()

                # 芯片大小不能超过设定限制
                if size > 1700:
                    return (0, 100000, 100000)

                # barcode有重复
                if len(library_data['barcode'].values) < len(set(library_data['barcode'].values)):
                    return (0, 100000, 100000)

                # 不平衡文库大于250G 不能添加
                if library_data[library_data['is_balance_lib'] == '否']['data_needed'].sum() > 250:
                    return (0, 100000, 100000)

                # 碱基不平衡不过不添加，保证前面的数据, 在数据达到1200G的时候开始
                ratio_sites, is_not_balance_list = count_barcode_radio(library_data)
                if is_not_balance_list:
                    return (0, 100000, 100000)

                if library_data[library_data['classification'].str.lower() == 'nextera']['data_needed'].sum() <= 50:
                    return (0, 100000, 100000)

                # 计算每个sheet的data_needed之和
                total_data_needed_sum += library_data['data_needed'].sum()

                # 记录包含字母"A"的sheet数量
                if any('极致' in value for value in library_data['split']):
                    xchip += 1
        except Exception:
            return (0, 100000, 100000)

        # 返回一个适应度值，目标是最大化总的data_needed之和，最小化sheet的数量, 最少的极致芯片
        total_data_needed_sum, num_sheets, num_xchip = total_data_needed_sum, len(individual[0]), xchip
        return total_data_needed_sum, num_sheets, num_xchip

    def run(self):
        # 定义遗传算法的参数
        pop_size = 50
        cxpb = 0.7  # 交叉概率
        mutpb = 0.2  # 变异概率
        ngen = 100  # 迭代次数

        # 初始化遗传算法工具箱
        creator.create("FitnessMax", base.Fitness, weights=(1.0, -1.0, -1.0,))  # 三个目标，一个最大化两个最小化
        creator.create("Individual", list, fitness=creator.FitnessMax)

        toolbox = base.Toolbox()

        # 结构初始化器
        toolbox.register("individual", tools.initRepeat, creator.Individual, self.generate_individual, n=3)
        toolbox.register("population", tools.initRepeat, list, toolbox.individual)
        toolbox.register("evaluate", self.evaluate)

        # 注册遗传算法所需的操作
        toolbox.register("mate", tools.cxTwoPoint)
        toolbox.register("mutate", tools.mutUniformInt, low=1, up=100, indpb=0.2)
        toolbox.register("select", tools.selTournament, tournsize=3)
        # 初始化种群
        population = toolbox.population(n=pop_size)

        # 运行遗传算法
        algorithms.eaMuPlusLambda(population, toolbox, mu=pop_size, lambda_=pop_size * 2, cxpb=cxpb, mutpb=mutpb,
                                  ngen=ngen, stats=None, halloffame=None)

        # 输出结果
        best_individual = tools.selBest(population, k=1)
        print(best_individual)
        optimized_sheets = best_individual[0]  # 获取最优解

        # 将优化后的结果输出
        # for i, sheet in enumerate(optimized_sheets):
        #     sheet.to_excel(f'optimized_sheet_{i + 1}.xlsx', index=False)
        return optimized_sheets


class AutoLayout:
    """
    自动化派样
    """

    def __init__(self, path, librarynum, output=basedir, data_limit=1750):
        self.path = path
        self.output = output
        self.librarynum = int(librarynum)
        self.data_limit = data_limit

        self.index_assignments = defaultdict(list)
        # 芯片数量量大小
        self.chip_size = dict()
        # 芯片是否极致
        self.chip_type = dict()
        # 芯片barcode
        self.chip_barcode_recode = defaultdict(set)
        # 芯片原始数据读取
        self.ori_data = self.read_excel()
        # 当前锚芯片
        self.loc_chip_num = 1
        # 芯片客户
        self.chip_customer = defaultdict(set)
        # 文库
        self.chip_classification = defaultdict(set)
        self.rule = self.read_rule()
        # 甲基化文库不大于200,WGBS文库不大于200G
        self.chip_speciallib_size = dict()

        # Nextera 文库大小
        self.chip_speciallib_nextera_size = dict()

        self.logger = log(os.path.basename(f'{path}.txt'))
        self.return_log = list()
        self.no_assign_data = list()
        self.need_cols = self.read_cols()

    def read_excel(self):
        """
        原始数据处理
        :return:
        """
        merge = pd.read_excel(self.path, None)
        ori_data = dict()
        for name, sheet in merge.items():
            sheet.fillna('', inplace=True)
            ori_data[name] = sheet.to_dict('records')
        return ori_data

    def add_new_data(self, chipname, library_data, newer=True):
        """
        增加新数据到已知芯片上
        :param chipname:
        :param library_data:
        :param newer:
        :return:
        """
        self.index_assignments[chipname].extend(library_data['data'])
        self.chip_barcode_recode[chipname].update({item['barcode'] for item in library_data['data']})

        self.chip_customer[chipname].add(library_data['customer'])
        self.chip_classification[chipname].add(library_data['classification'])

        if newer:
            self.chip_size[chipname] = library_data['size']
            # if library_data['classification'] in ['扩增子', '不平衡文库', '单细胞文库以及甲基化']:
            if library_data['is_balance_lib'] == '否':
                self.chip_speciallib_size[chipname] = library_data['size']
            else:
                self.chip_speciallib_size[chipname] = 0
            if 'nextera' in library_data['classification'].lower():
                self.chip_speciallib_nextera_size[chipname] = library_data['size']
            else:
                self.chip_speciallib_nextera_size[chipname] = 0

        else:
            self.chip_size[chipname] += library_data['size']
            if library_data['is_balance_lib'] == '否':
                self.chip_speciallib_size[chipname] += library_data['size']
            if 'nextera' in library_data['classification'].lower():
                self.chip_speciallib_nextera_size[chipname] += library_data['size']

    @staticmethod
    def count_barcode_radio(data):
        df = pd.DataFrame(data)
        ratio_sites = dict()
        is_not_balance_list = []
        if df.empty:
            return ratio_sites, is_not_balance_list

        df['barcode'] = df['barcode'].str.slice(0, 16)
        barcode_df = pd.DataFrame(df['barcode'].str.split('', expand=True).iloc[:, 1:-1].values,
                                  columns=['T' + str(x) for x in range(16)]).join(df['data_needed'])
        total = barcode_df['data_needed'].sum()

        for i in range(16):
            column = 'T' + str(i)
            col_df = barcode_df.groupby(column).agg({'data_needed': 'sum'})
            # 去掉N计数
            if 'N' in col_df.index:
                base_n_size = col_df.loc['N', 'data_needed']
                col_df = col_df.drop('N')
            else:
                base_n_size = 0
            col_df['ratio'] = (col_df['data_needed']) / (total - base_n_size)
            ratio = col_df['ratio'].to_dict()
            ratio_sites[i] = ratio
            A, B, C, D, E, F = list(), list(), list(), list(), list(), list()
            for decbase in ['A', 'T', 'C', 'G']:
                if decbase not in ratio:
                    ratio[decbase] = 0
                if ratio[decbase] >= 0.6:
                    A.append(decbase)
                if 0.2 <= ratio[decbase] < 0.6:
                    B.append(decbase)
                if 0.15 <= ratio[decbase] < 0.2:
                    C.append(decbase)
                if 0.1 <= ratio[decbase] < 0.15:
                    D.append(decbase)
                if 0.08 <= ratio[decbase] < 0.1:
                    E.append(decbase)
                if ratio[decbase] < 0.08:
                    F.append(decbase)

            A_num, B_num, C_num, D_num, E_num, F_num = len(A), len(B), len(C), len(D), len(E), len(F)
            if not ((B_num + C_num + D_num == 4) or (F_num == 1 and (A_num + B_num) == 3) or (
                    E_num == 1 and D_num == 1 and (A_num + B_num + C_num) == 2) or (
                            E_num == 1 and (A_num + B_num + C_num) == 3)):
                is_not_balance_list.append(
                    '第%s位置，算出结果为 %s' % (i, ratio)
                )
        return ratio_sites, is_not_balance_list

    def dec_barcode_radio(self, chipname):
        data = self.index_assignments[chipname]
        ratio_sites, is_not_balance_list = self.count_barcode_radio(data)
        if is_not_balance_list:
            desc = '\n'.join(is_not_balance_list)
            self.return_log.append(f'芯片{chipname}有碱基不平衡:\n{desc}')
            print(f'芯片{chipname}有碱基不平衡:\n{desc}')

    @staticmethod
    def level(row):

        today_date = datetime.now()

        # 将时间字符串转换为 datetime 对象
        # mytime = datetime.strptime(row['time'], "%Y-%m-%d")
        # mytime = row['time'].strftime("%Y-%m-%d")

        if 'nextera' in row['classification'].lower():
            return 1

        if row['拆分方式'] == '极致周期' or '极致' in row['拆分方式']:
            return 2

        mytime = row['time']
        # 判断日期是之前的还是之后的
        if mytime < today_date:
            return 3

        if '加急' in row['priority']:
            return 4

        if '补测' in row['priority']:
            return 5

        else:
            return 100

    @staticmethod
    def read_rule():
        df = pd.read_excel(os.path.join(basedir, 'rule', 'exclusive_classfication.xlsx'))
        newdf = pd.DataFrame()
        newdf['c1'] = df['c2']
        newdf['c2'] = df['c1']
        res = pd.concat([df, newdf])
        return res.reset_index()

    @staticmethod
    def read_cols():
        df = pd.read_excel(os.path.join(basedir, 'rule', 'columns.xlsx'))
        cols = list(df['cols'].values)
        return cols

    def use_rule(self, chipname, classfication):
        may_classfic = set(self.rule[self.rule['c1'] == classfication]['c2'])
        if self.chip_customer[chipname].intersection(may_classfic):
            return True
        return False

    def judge_data(self, chipname, library_data):
        size = library_data['size']
        # customer = library_data['customer']
        # library = library_data['library']
        classification = library_data['classification']
        is_balance_lib = library_data['is_balance_lib']

        # 芯片大小不能超过设定限制
        sizelimit = True
        if self.chip_size[chipname] + size > self.data_limit:
            sizelimit = False

        # barcode有重复
        notrepeatbarcode = True
        if self.chip_barcode_recode[chipname].intersection({item['barcode'] for item in library_data['data']}):
            notrepeatbarcode = False

        # 互斥的文库
        exclusive_classific = True
        if self.use_rule(chipname, classification):
            exclusive_classific = False

        # 不平衡文库大于250G 不能添加
        splibrary = True
        if is_balance_lib == '否' and self.chip_speciallib_size[chipname] + size > 250:
            splibrary = False

        # 碱基不平衡不过不添加，保证前面的数据, 在数据达到1200G的时候开始
        base_balance = True
        if self.chip_size[chipname] > 800:
            current_data = copy.deepcopy(self.index_assignments[chipname])
            new_data = library_data['data']
            current_data.extend(new_data)
            ratio_sites, is_not_balance_list = self.count_barcode_radio(current_data)
            if is_not_balance_list:
                base_balance = False

        if sizelimit and notrepeatbarcode and exclusive_classific and splibrary and base_balance:
            return True
        return False

    def add_loc_num(self):
        # 有nextera文库 必须满足大于50G
        chipname = f'chip{self.loc_chip_num}'
        nextera_size = self.chip_speciallib_nextera_size[chipname]
        if nextera_size > 50 or nextera_size == 0:
            self.loc_chip_num += 1
        else:
            # 有nextera文库，但是不满足50G 去除
            nextary_barcode = set()
            no_nextary_data = list()
            for libdata in self.index_assignments[chipname]:
                if libdata['classification'].lower() != 'nextera':
                    no_nextary_data.append(libdata)
                else:
                    self.no_assign_data.append(libdata)
                    nextary_barcode.update(libdata['barcode'])
            self.index_assignments[chipname] = no_nextary_data
            self.chip_barcode_recode[chipname] -= nextary_barcode
            self.chip_speciallib_nextera_size[chipname] = 0

    def assign_samples(self):
        ori_library_data = list()

        if '未测' not in self.ori_data.keys():
            raise UserWarning('提供excel没有 未测 sheet ,请核查！')
        ori_library_df = pd.DataFrame(self.ori_data['未测'])

        # need_col = ['status', '#library', 'sublibrary', 'i5', 'i7', 'data_needed', 'real_data', 'customer',
        #             'classification', 'priority', 'time', '拆分方式', 'barcode', 'is_balance_lib', '备注',
        #             'TIPS1', 'TIPS2', 'TIPS3'
        #             ]
        self.need_cols = self.read_cols()
        get_col = set(ori_library_df.columns)
        unhave_col = set(self.need_cols) - get_col

        if unhave_col:
            unhave_fom = '; '.join(unhave_col)
            raise UserWarning(f'未测表里没有{unhave_fom}  表头,请核查！')

        numeric_mask = pd.to_numeric(ori_library_df['data_needed'], errors='coerce').notna()
        time_mask = pd.to_datetime(ori_library_df['time'], errors='coerce').notna()

        # 添加处理status列的逻辑
        status_mask = ori_library_df['status'] == '暂不排样'

        ori_library_df['note'] = ''
        ori_library_df.loc[~numeric_mask, 'note'] = 'data_needed 列非数字'
        ori_library_df.loc[~time_mask, 'note'] = 'time 列非日期'
        ori_library_df.loc[status_mask, 'note'] = '暂不排样'

        # need_col.append('note')

        no_ori_data = ori_library_df[~(numeric_mask & time_mask) | status_mask]

        self.no_assign_data.extend(no_ori_data.to_dict('records'))

        # 使用布尔索引筛选出不是数字和非日期的行
        ori_library_df = ori_library_df[(numeric_mask & time_mask) & ~status_mask]

        ori_library_df['level'] = ori_library_df.apply(self.level, axis=1)

        # # 极致客户有重复的，把等级调到0，防止放到了最后，到了未测里
        # duplicate_name = ori_library_df[ori_library_df['level'] == 2].duplicated(subset='barcode', keep=False)
        # # 将 'level' 列的值改为 0
        # ori_library_df.loc[duplicate_name, 'level'] = 0

        for library, library_df in ori_library_df.groupby('#library'):

            size = library_df['data_needed'].sum()

            # 文库内部有重复
            if len(library_df['barcode'].values) > len(set(library_df['barcode'].values)):
                library_df['note'] = '文库内部有重复'
                library_df.loc[:, 'time'] = library_df['time'].apply(format_date)
                self.no_assign_data.extend(library_df.to_dict('records'))
                continue

            flag = False
            if size > (self.data_limit) / 2:
                library_df['data_needed'] = library_df['data_needed'] / 2
                flag = True

            ori_library_data.append(dict(
                library=library,
                is_balance_lib=library_df['is_balance_lib'].values[0],
                size=library_df['data_needed'].sum(),
                split_method=library_df['拆分方式'].values[0],
                time=library_df['time'].values[0],
                level=library_df['level'].values[0],
                customer=library_df['customer'].values[0],
                classification=library_df['classification'].values[0],
                data=library_df[self.need_cols].to_dict('records')
            ))

            if flag:
                self.return_log.append(f'文库{library} 已做拆分处理, 请注意！！！ ')
                ori_library_data.append(dict(
                    library=library,
                    is_balance_lib=library_df['is_balance_lib'].values[0],
                    size=library_df['data_needed'].sum(),
                    split_method=library_df['拆分方式'].values[0],
                    time=library_df['time'].values[0],
                    level=library_df['level'].values[0],
                    customer=library_df['customer'].values[0],
                    classification=library_df['classification'].values[0],
                    data=library_df[self.need_cols].to_dict('records')
                ))
        ori_sort_data = sorted(ori_library_data, key=lambda x: (x['level'], x['time'], -x['size']))

        i = 0
        while ori_sort_data:
            library_data = ori_sort_data[0]
            chipname = f'chip{self.loc_chip_num}'

            # 空白芯片直接添加
            if chipname not in self.index_assignments:
                self.add_new_data(chipname, library_data)
                ori_sort_data.remove(library_data)
                i += 1
                continue

            # 判断条件
            if self.judge_data(chipname, library_data):
                self.add_new_data(chipname, library_data, newer=False)
                ori_sort_data.remove(library_data)
                i += 1
            else:
                for j in range(len(ori_sort_data)):
                    newlibrary_data = ori_sort_data[j]
                    if self.judge_data(chipname, newlibrary_data):
                        ori_sort_data.remove(newlibrary_data)
                        i += 1
                        self.add_new_data(chipname, newlibrary_data, newer=False)
                        break
                    j += 1
                else:
                    # self.loc_chip_num += 1
                    self.add_loc_num()
            if self.chip_size[chipname] > self.data_limit:
                # self.loc_chip_num += 1
                self.add_loc_num()

    def assign_again(self):
        pass

    def run(self):
        # self.assign_samples()
        try:
            self.assign_samples()
        except Exception as e:
            self.return_log.append(f'T7排样出错， 请联系！{e}')
            self.index_assignments = {}
        outputname = 'assignments_%s_%s' % (datetime.now().strftime("%m%d%H%M"), os.path.basename(self.path))
        outputpath = os.path.join(self.output, 'result', outputname)
        writer = pd.ExcelWriter(outputpath)

        # ga = Ga(sheets=self.index_assignments)
        # self.index_assignments = ga.run()

        chip_loc = 1
        librarynum = 0
        for chip_idx, chip_assignments in self.index_assignments.items():
            if not chip_assignments:
                continue
            df = pd.DataFrame(chip_assignments)
            df['time'] = df['time'].dt.strftime('%Y-%m-%d')

            if df['data_needed'].sum() < 1600 or librarynum > self.librarynum:
                df['note'] = '排样数据量不足1600或者排样管数超标'
                self.no_assign_data.extend(df.to_dict('records'))
                continue
            librarynum += len(set(df['#library'].values))
            if [method for method in df['拆分方式'].values if '极致' in method]:
                addname = 'X'
            else:
                addname = ''
            self.dec_barcode_radio(chip_idx)
            chipname = addname + chip_idx

            sum_list = list()
            for library, library_df in df.groupby('#library'):
                sum_list.append(dict(
                    二次拆分=library,
                    客户=library_df['customer'].values[0],
                    类型=library_df['classification'].values[0],
                    打折前=library_df['data_needed'].sum()
                ))
            df_sum = pd.DataFrame(sum_list)
            res_df = pd.concat([df, df_sum], axis=1)
            res_df.to_excel(writer, sheet_name=chipname, index=False)
            chip_loc += 1
        no_assign_df = pd.DataFrame(self.no_assign_data)
        no_assign_df = no_assign_df.applymap(lambda x: format_date(x) if isinstance(x, pd.Timestamp) else x)
        no_assign_df = no_assign_df[self.need_cols]
        no_assign_df.to_excel(writer, sheet_name='未测', index=False)
        if self.return_log:
            pd.DataFrame(self.return_log).to_excel(writer, sheet_name='log', index=False)
        writer.close()
        return outputpath


if __name__ == '__main__':
    start_time = time.time()
    filepath = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'example', 'input排样表.xlsx')
    # excel_file = 'example/input排样表.xlsx'
    output_file = ''
    layout = AutoLayout(filepath, output_file)
    layout.run()
    end_time = time.time()
    execution_time = end_time - start_time
    print(f"代码执行时间为：{execution_time} 秒")

    # server()