import cvxpy as cp import copy from numpy import linalg as LA import os import traceback import glob import numpy as np import pandas as pd import math from ..signal.engine import * from ..signal.interface import * from ..risk import * from ..domain import * # === Axioma specific covariance integration === from model.trading.process_axioma_data import process_axioma_specific_cov AXIOMA_DATA_DIR = "./axioma_data" from model.trading.process_gqr_specret import process_gqr_specret # Set pandas display options pd.set_option('display.max_rows', None) # Show all rows pd.set_option('display.max_columns', None) # Show all columns pd.set_option('display.width', 1000) # Set wider console width pd.set_option('display.max_colwidth', None) # Show full content in each column pd.set_option('display.float_format', '{:.4f}'.format) # Show floating point numbers with 4 decimals def matrix_shrink(Q): """ Take Q and reapprox using subset of singular values """ # shrink matrix U, S, V = LA.svd(Q) num_sing_values = S.shape[0] U = U[:, :num_sing_values] # log some singular values PyLog.info(f"matrix_shrink sing values top5 {S[0:5]} bot5 {S[-5:]}") # use at least portion of the singular values, and always at least K but not more than the rank of the matrix # very cut down settings we have used in some tests: portion= 1/8; K = 10 # typical keep most settings portion= 7/8; K= 20 portion = 7/8 K = 20 num_sing_vaule_to_use = max(int(num_sing_values*portion), K) r = min(num_sing_values, num_sing_vaule_to_use) PyLog.info(f"matrix_shrink got {num_sing_values} singular values. portion:{portion:.3f} K:{K} reconstructing with {r}. matrix shpe {Q.shape} first/last {S[0]/S[4] :.2f}") S = np.diag(S) R = U[:, :r] @ S[:r, :r] @ V[:r, :] return R class RebalConfig: dctDefaultOptParams = { 'SCS': {'maxIter': 200_000, 'cycleIter': 5_000, 'targetTolerance': 1e-4, 'toleranceRange': [7e-4, 1e-3, 2e-3, 3e-3, 4e-3, 5e-3]}, 'ECOS': {'maxIter': 20_000} } DEFAULT_CONFIG = {'configName': 'DEFAULT', 'stratName': 'strat_asia_v1', 'maxLeverage': +4.00, 'minNetExposure': -0.05, 'maxNetExposure': +0.40, 'minBetaAdjNet': -0.01, 'maxBetaAdjNet': +0.01, 'gamma': 1.8, 'adaptiveGamma': False, 'adaptiveGammaTargetRisk': 0.10, 'adaptiveGammaTolerance': 0.0025, 'tau': 4.0, 'maxRisk': 0.13, 'offDiagRisk': False, 'splitLongShort': False, 'splitLongShortStartDate': PyDate.asDate(20220615), 'bcostLookback': 8, 'maxShortUtilization': 0.8, 'pbList': None, 'bcostMultiplierName': None, 'HAUM': False, 'hypotheticalAUM': 100_000_000, 'univwtThreshold': 0.25, 'holdingThreshold': 0.00005, 'rmodelName': 'risk_model', 'tmodelName': 'tcost_model', 'impactCostScale': 1.0, 'alphaName': 'alpha', 'tradeUniverseName': None, 'themeBoundName': 'trading_alpha_theme_exposure_bounds', 'fcostName': 'trading_long_financing_cost', 'bcostName': 'trading_short_borrow_cost', 'defaultFCost': 0.005, 'defaultBCost': 0.15, 'longBoundName': 'trading_position_bounds_long', 'shortBoundName': 'trading_position_bounds_short', 'defaultLongBound': 0.000025, 'defaultShortBound': 0.000025, 'liquidityName': 'adv_usdm_blended', 'defaultLiquidityUSDM': 1.0, 'maxLiquidityBoundLong': None, 'maxLiquidityBoundShort': None, 'applyShortBan': True, 'applyExchangeHolidays': False, 'advName': 'adv_usdm_blended', 'countryBoundName': 'trading_country_bounds_frame', 'industryBoundName': 'trading_industry_bounds_frame', 'sizeBoundName': 'trading_size_bounds_frame', 'targetTolerance': 1e-4, 'toleranceRange': [7e-4, 1e-3, 2e-3, 3e-3, 4e-3, 5e-3], 'minShortAvailM': 0, 'shrinkfcov' : False, 'enable_gs_stability_score' : False, 'tightCountry' : False,} @classmethod def defaultConfig(cls, parameter): return cls.DEFAULT_CONFIG[parameter] def __init__(self, configName = None, stratName = None, maxLeverage = None, minNetExposure = None, maxNetExposure = None, minBetaAdjNet = None, maxBetaAdjNet = None, gamma = None, adaptiveGamma = None, adaptiveGammaTargetRisk = None, adaptiveGammaTolerance = None, tau = None, maxRisk = None, offDiagRisk = None, splitLongShort = None, splitLongShortStartDate = None, maxShortUtilization = None, pbList = None, bcostMultiplierName = None, HAUM = None, hypotheticalAUM = None, bcostLookback = None, univwtThreshold = None, holdingThreshold = None, rmodelName = None, tmodelName = None, impactCostScale = None, alphaName = None, tradeUniverseName = None, themeBoundName = None, fcostName = None, bcostName = None, defaultFCost = None, defaultBCost = None, longBoundName = None, shortBoundName = None, liquidityName = None, defaultLiquidityUSDM = None, maxLiquidityBoundLong = None, maxLiquidityBoundShort = None, defaultLongBound = None, defaultShortBound = None, applyShortBan = None, applyExchangeHolidays = None, advName = None, maxAdvProp = None, countryBoundName = None, industryBoundName = None, sizeBoundName = None, targetTolerance = None, toleranceRange = None, minShortAvailM = None, shrinkfcov = None, enable_gs_stability_score = None, tightCountry = None,): self.configName = self.defaultConfig('configName') if configName is None else configName self.stratName = self.defaultConfig('stratName') if stratName is None else stratName self.maxLeverage = self.defaultConfig('maxLeverage') if maxLeverage is None else maxLeverage self.minNetExposure = self.defaultConfig('minNetExposure') if minNetExposure is None else minNetExposure self.maxNetExposure = self.defaultConfig('maxNetExposure') if maxNetExposure is None else maxNetExposure self.minBetaAdjNet = self.defaultConfig('minBetaAdjNet') if minBetaAdjNet is None else minBetaAdjNet self.maxBetaAdjNet = self.defaultConfig('maxBetaAdjNet') if maxBetaAdjNet is None else maxBetaAdjNet self.gamma = self.defaultConfig('gamma') if gamma is None else gamma self.adaptiveGamma = self.defaultConfig('adaptiveGamma') if adaptiveGamma is None else adaptiveGamma self.adaptiveGammaTargetRisk = self.defaultConfig('adaptiveGammaTargetRisk')\ if adaptiveGammaTargetRisk is None else adaptiveGammaTargetRisk self.adaptiveGammaTolerance = self.defaultConfig('adaptiveGammaTolerance')\ if adaptiveGammaTolerance is None else adaptiveGammaTolerance self.tau = self.defaultConfig('tau') if tau is None else tau self.maxRisk = self.defaultConfig('maxRisk') if maxRisk is None else maxRisk self.offDiagRisk = self.defaultConfig('offDiagRisk') if offDiagRisk is None else offDiagRisk self.splitLongShort = self.defaultConfig('splitLongShort') if splitLongShort is None else splitLongShort self.splitLongShortStartDate = self.defaultConfig('splitLongShortStartDate')\ if splitLongShortStartDate is None else splitLongShortStartDate self.maxShortUtilization = self.defaultConfig('maxShortUtilization')\ if maxShortUtilization is None else maxShortUtilization self.pbList = self.defaultConfig('pbList') if pbList is None else pbList self.bcostMultiplierName = self.defaultConfig('bcostMultiplierName')\ if bcostMultiplierName is None else bcostMultiplierName self.HAUM = self.defaultConfig('HAUM') if HAUM is None else HAUM self.hypotheticalAUM = self.defaultConfig('hypotheticalAUM')\ if hypotheticalAUM is None else hypotheticalAUM self.bcostLookback = self.defaultConfig('bcostLookback') if bcostLookback is None else bcostLookback self.univwtThreshold = self.defaultConfig('univwtThreshold') if univwtThreshold is None else univwtThreshold self.holdingThreshold = self.defaultConfig('holdingThreshold') if holdingThreshold is None else holdingThreshold self.rmodelName = self.defaultConfig('rmodelName') if rmodelName is None else rmodelName self.tmodelName = self.defaultConfig('tmodelName') if tmodelName is None else tmodelName self.impactCostScale = self.defaultConfig('impactCostScale') if impactCostScale is None else impactCostScale self.alphaName = self.defaultConfig('alphaName') if alphaName is None else alphaName self.tradeUniverseName = \ self.defaultConfig('tradeUniverseName') if tradeUniverseName is None else tradeUniverseName self.themeBoundName = themeBoundName self.fcostName = self.defaultConfig('fcostName') if fcostName is None else fcostName self.bcostName = self.defaultConfig('bcostName') if bcostName is None else bcostName self.defaultFCost = self.defaultConfig('defaultFCost') if defaultFCost is None else defaultFCost self.defaultBCost = self.defaultConfig('defaultBCost') if defaultBCost is None else defaultBCost self.longBoundName = self.defaultConfig('longBoundName') if longBoundName is None else longBoundName self.shortBoundName = self.defaultConfig('shortBoundName') if shortBoundName is None else shortBoundName self.defaultLongBound = \ self.defaultConfig('defaultLongBound') if defaultLongBound is None else defaultLongBound self.defaultShortBound = \ self.defaultConfig('defaultShortBound') if defaultShortBound is None else defaultShortBound self.liquidityName = \ self.defaultConfig('liquidityName') if liquidityName is None else liquidityName self.defaultLiquidityUSDM = \ self.defaultConfig('defaultLiquidityUSDM') if defaultLiquidityUSDM is None else defaultLiquidityUSDM self.maxLiquidityBoundLong = \ self.defaultConfig('maxLiquidityBoundLong') if maxLiquidityBoundLong is None else maxLiquidityBoundLong self.maxLiquidityBoundShort = \ self.defaultConfig('maxLiquidityBoundShort') if maxLiquidityBoundShort is None else maxLiquidityBoundShort self.applyShortBan = self.defaultConfig('applyShortBan') if applyShortBan is None else applyShortBan self.applyExchangeHolidays = \ self.defaultConfig('applyExchangeHolidays') if applyExchangeHolidays is None else applyExchangeHolidays self.advName = self.defaultConfig('advName') if advName is None else advName self.maxAdvProp = maxAdvProp self.countryBoundName = \ self.defaultConfig('countryBoundName') if countryBoundName is None else countryBoundName self.industryBoundName = \ self.defaultConfig('industryBoundName') if industryBoundName is None else industryBoundName self.sizeBoundName = self.defaultConfig('sizeBoundName') if sizeBoundName is None else sizeBoundName self.targetTolerance = self.defaultConfig('targetTolerance') if targetTolerance is None else targetTolerance self.toleranceRange = self.defaultConfig('toleranceRange') if toleranceRange is None else toleranceRange self.minShortAvailM = self.defaultConfig('minShortAvailM') if minShortAvailM is None else minShortAvailM self.shrinkfcov = self.defaultConfig('shrinkfcov') if shrinkfcov is None else shrinkfcov self.tightCountry = self.defaultConfig('tightCountry') if tightCountry is None else tightCountry self.enable_gs_stability_score = self.defaultConfig('enable_gs_stability_score') if enable_gs_stability_score is None else enable_gs_stability_score RebalConfig_DEFAULT = RebalConfig() ## ================================================================================================================== class Rebalance: EMPTY_PORTFOLIO = pd.Series(1.0, index=['$USD']) @classmethod def saveSimScratch(cls, df, date=None, name_without_path=None): """ hack method for auditing and analysis drop the df in the sim scratch directory for post inspection place a call whereever you want a dataframe for analysis, eg cls.saveSimScratch(dframe, tradeDate, "weights_early") args df: dataframe to save down date : if found appends to name name_without_path : no path, no extension """ user = os.environ['USER'] base = f"/home/{user}/scratch/r" os.makedirs(base, exist_ok=True) if name_without_path is None: name_without_path = "scratch_file" if date: date_str = date.strftime("%Y%m%d") filebase = f"{base}/{name_without_path}_{date_str}" else: filebase = f"{base}/{name_without_path}" df.to_parquet(f"{filebase}.pq") df.to_csv(f"{filebase}.csv") @classmethod def run(cls, rebalConfig=RebalConfig_DEFAULT, signalDate=PyDate.asDate(20191115), preOptWeights=EMPTY_PORTFOLIO, NAV=100000000, solver=cp.SCS, maxIter=200000, cycleIter=5000, verbose=True, checkDCP=True, optimal=False, tradeRestrictions=None): tradeDate = PyDate.nextWeekday(signalDate) PyLog.info('optimizing for SD:{} / TD:{}'.format(PyDate.asISO(signalDate), PyDate.asISO(tradeDate))) dctOptResult = dict() dctOptResult['rebalConfig'] = rebalConfig dctOptResult['signalDate'] = signalDate dctOptResult['preOptNAV'] = NAV if len(preOptWeights) == 0: dfWeights = pd.DataFrame({'assetKey': ['assetKey'], 'preOptWeights': [0.0]}).iloc[:0] else: dfWeights = preOptWeights.reset_index().rename(columns={'index': 'assetKey', 0: 'preOptWeights'}) dfWeights = dfWeights[[Real.isNonZero(x) for x in dfWeights['preOptWeights']]] stratName = rebalConfig.stratName rmodel = SignalMgr.get(rebalConfig.rmodelName, signalDate, stratName) tmodel = SignalMgr.get(rebalConfig.tmodelName, signalDate, stratName).set_index('assetKey') # gmult = Trading.getModelParameters('rrmult', signalDate) PyLog.info("Rebalance.py.run SET GMULT=1.0 --FROM YB TRADING GOOGLE SHEET") gmult=1.0 PyLog.info("Rebalance.py.run SET TCAFMULT=1.0 --FROM YB TRADING GOOGLE SHEET") tcafmult =1.0 # tcafmult = Trading.getModelParameters('tcafmult', signalDate) signals = {rebalConfig.alphaName : 'alpha', rebalConfig.fcostName : 'fcost', rebalConfig.bcostName : 'bcost', rebalConfig.longBoundName : 'longBoundSoft', rebalConfig.shortBoundName : 'shortBoundSoft', 'latest_mktcap_usdm' : 'mktcap', 'univwt' : 'univwt', 'model_country' : 'modelCountry'} dframe = SignalMgr.getFrame(list(signals.keys()), signalDate, stratName).\ drop(columns=['signalDate']).rename(columns=signals).set_index('assetKey') tmult = SignalMgr.get('trading_tcost_amortization_factor_multiplier', signalDate, stratName).iloc[0] dct = {'tcafmult': tcafmult, 'tmult': tmult} ## hard position bounds --------------------------------------------------------------------------------- dframe = dframe.assign(longBoundHard = 1.0) dframe = dframe.assign(shortBoundHard = 1.0) dframe = dframe[dframe['univwt'] >= rebalConfig.univwtThreshold] dframe = dframe[~dframe['alpha'].isnull()] if rebalConfig.tradeUniverseName is not None: PyLog.info(f"Getting rebalConfig.tradeUniverseName:{rebalConfig.tradeUniverseName} signalDate:{signalDate} stratName:{stratName}") universe = SignalMgr.get(rebalConfig.tradeUniverseName, signalDate, stratName) dframe_sz_was = len(dframe) dframe = dframe[dframe.index.isin(universe.index)] dframe_sz_is = len(dframe) PyLog.info(f"dframe filtered from {dframe_sz_was} to {dframe_sz_is}. universe size:{len(universe)}") ## drop assets that are excluded from both long and short bounds dframe = dframe[~dframe['longBoundSoft'].isnull() | ~dframe['shortBoundSoft'].isnull()] # assets = list((set(rmodel['assets']).intersection(set(dframe.index)).intersection(set(tmodel.index))).\ # union(set(dfWeights['assetKey']))) assets = set(rmodel['assets']).intersection(set(dframe.index)) assets = assets.intersection(set(tmodel.index)) assets = sorted(list(assets)) dframe = dframe.reindex(index=assets).reset_index() dframe = dframe.assign(fcost = dframe['fcost'].fillna(rebalConfig.defaultFCost)) dframe = dframe.assign(bcost = dframe['bcost'].fillna(rebalConfig.defaultBCost)) ## position bounds ------------------------------------------------------------------------------------- dframe = dframe.assign(longBoundSoft = dframe['longBoundSoft'].fillna(rebalConfig.defaultLongBound)) dframe = dframe.assign(shortBoundSoft = dframe['shortBoundSoft'].fillna(rebalConfig.defaultShortBound)) ## in addition, we want to restrict short to 2% of market cap dframe = dframe.assign(shortAvail = dframe['mktcap'].fillna(10.0) * 0.02 * 1000000 / NAV) ## cut on min shortAvail in M;- i.e. zero out shortAvail if number is 0]) )) dframe.loc[dframe.shortAvail < minShortAvailM, 'shortAvail'] = 0 # dframe = dframe.assign(shortBound = dframe[['shortBound', 'shortAvail']].min(axis=1)) dframe = dframe.assign(shortBoundHard = dframe[['shortBoundHard', 'shortAvail']].min(axis=1)) dframe = dframe.drop(columns='shortAvail') ## apply liquidity-based position bounds --------------------------------------------------------------- if (rebalConfig.maxLiquidityBoundLong is not None) or (rebalConfig.maxLiquidityBoundShort is not None): dfm = SignalMgr.getFrame(rebalConfig.liquidityName, signalDate, stratName) dframe = dframe.merge(dfm[['assetKey', rebalConfig.liquidityName]], how='left', on='assetKey') dframe[rebalConfig.liquidityName] = \ dframe[rebalConfig.liquidityName].fillna(rebalConfig.defaultLiquidityUSDM) if rebalConfig.maxLiquidityBoundLong is not None: dframe = dframe.assign( liquidityBoundLong = rebalConfig.maxLiquidityBoundLong * dframe[rebalConfig.liquidityName] * 1000000 / NAV) dframe = dframe.assign(longBoundSoft = dframe[['longBoundSoft', 'liquidityBoundLong']].min(axis=1)) dframe = dframe.drop(columns=['liquidityBoundLong']) if rebalConfig.maxLiquidityBoundShort is not None: dframe = dframe.assign( liquidityBoundShort = rebalConfig.maxLiquidityBoundShort * dframe[rebalConfig.liquidityName] * 1000000 / NAV) dframe = dframe.assign(shortBoundSoft = dframe[['shortBoundSoft', 'liquidityBoundShort']].min(axis=1)) dframe = dframe.drop(columns=['liquidityBoundShort']) dframe = dframe.drop(columns=[rebalConfig.liquidityName]) ## ----------------------------------------------------------------------------------------------------- ## apply short bans ------------------------------------------------------------------------------------ if rebalConfig.applyShortBan: if PyDate.ge(signalDate, PyMonth.firstWeekday(202003)): if PyDate.le(signalDate, PyMonth.firstWeekday(202105)): countries = ['KR', 'ID'] else: countries = ['ID'] dframe = dframe.assign( preOptWeights = Filter.bound(-preOptWeights.reindex(dframe['assetKey']).fillna(0.0), lower=0.0)) dframe.loc[dframe['modelCountry'].isin(countries), 'preOptWeights'] =\ dframe.loc[dframe['modelCountry'].isin(countries), 'preOptWeights'] dframe.loc[~dframe['modelCountry'].isin(countries), 'preOptWeights'] = 1.0 # dframe = dframe.assign(shortBound = dframe[['shortBound', 'preOptWeights']].min(axis=1)) dframe = dframe.assign(shortBoundHard = dframe[['shortBoundHard', 'preOptWeights']].min(axis=1)) dframe = dframe.drop(columns='preOptWeights') ## ----------------------------------------------------------------------------------------------------- # === KR SHORTS ZERO-OUT (no KR shorts) ======================================= #### KR shorting is not working in the optimizer, we have consistently lost money since the short # ban was lifted in April 2025, so below code disable KR short selling. This is temporary till # we figure out what is the core issue with KR shorts # Policy: immediately cover any KR short; forbid new KR shorts. KR longs unaffected. try: # Detect Korea by modelCountry; also accept RIC suffixes (.KS = KOSPI, .KQ = KOSDAQ) if present is_kr = dframe['modelCountry'].astype(str).str.upper().eq('KR') if 'ric' in dframe.columns: is_kr = is_kr | dframe['ric'].astype(str).str.upper().str.endswith(('.KS', '.KQ')) is_kr = is_kr.fillna(False).to_numpy() # Asset-level: set shortBoundHard to 0 (no shorts). Keep longBoundHard unchanged. sbh = dframe['shortBoundHard'].astype(float).fillna(0.0).to_numpy() sbh[is_kr] = 0.0 dframe['shortBoundHard'] = sbh # Per-availability (splitLongShort=True): also clamp line-level short bounds to 0 if 'assetKey' in dfAvail.columns and 'shortBoundHard' in dfAvail.columns: kr_keys = set(dframe.loc[is_kr, 'assetKey']) if len(kr_keys) > 0: mask_av = dfAvail['assetKey'].isin(kr_keys) dfAvail.loc[mask_av, 'shortBoundHard'] = 0.0 PyLog.info(f"KR shorts disabled: set shortBoundHard=0 for {int(is_kr.sum())} KR names (asset + availability).") except Exception as e: PyLog.info(f"KR shorts disable not applied: {e}") # === end KR SHORTS ZERO-OUT ================================================== PyLog.info(f"dframe len:{len(dframe)} {dframe.columns.tolist()}") ######## Go impose TradeRestrictions pre-optimiser on longBoundHard/ShortBoundHard if(tradeRestrictions): from .TradeRestrictions import TradeRestrictions #fn_ob = 'preOpt_preTR.{}.csv'.format(str(tradeDate).replace('-','')); sim_dir = '/mnt/signal/simulation/EG001'; fn_ob = os.path.join(f"{sim_dir}/{fn_ob}") #dframe.to_csv(fn_ob, index=False, float_format='%.10f') #PyLog.info('tr.applyTradeRestrictions PRE-OPT ...go') added_preOptWeights=False if not 'preOptWeights' in dframe.columns: preOptWeights_df = pd.DataFrame({'assetKey':preOptWeights.index, 'preOptWeights':preOptWeights.values}) dframe = dframe.merge(preOptWeights_df, how='left', on='assetKey').fillna(0.0) added_preOptWeights=True dframe = tradeRestrictions.applyTradeRestrictions( tradeDate, dframe, NAV) #fn_ob = 'preOpt_postTR.{}.csv'.format(str(tradeDate).replace('-','')); sim_dir = '/mnt/signal/simulation/EG001'; fn_ob = os.path.join(f"{sim_dir}/{fn_ob}") #dframe.to_csv(fn_ob, index=False, float_format='%.10f') if added_preOptWeights and 'preOptWeights' in dframe.columns: dframe.drop(['preOptWeights'], axis=1, inplace=True) # ############### Done impose TradeRestrictions ###################################################### ## dframe is what goes into optimization / dfWeights is for record keeping dframe = dframe.merge(dfWeights, how='left', on='assetKey') PyLog.info(f"tradeDate: {tradeDate} rebalConfig.splitLongShort:{rebalConfig.splitLongShort} rebalConfig.splitLongShortStartDate:{rebalConfig.splitLongShortStartDate}") ## Note, the time condition here is different from that in Simulation. We want the splitLongShort ## applied in optimization first before applying it in the portfolio accounting. if rebalConfig.splitLongShort and (tradeDate >= rebalConfig.splitLongShortStartDate): dfAvail, dfMarginal = cls.compileShortAvailability(rebalConfig, NAV, tradeDate, dframe) #this was allowing optimal starting positions to be constructed exceeding avail borrow. Commenting out. #if optimal: # dfMarginal = dfAvail.groupby('assetKey').agg({'shortBoundHard': sum}).reset_index().\ # rename(columns={'shortBoundHard': 'availablePct'}) dframe = dframe.merge(dfMarginal.rename(columns={'availablePct': 'shortBound'}), how='left', on='assetKey') dframe = dframe.assign(shortBound=Real.isNegative(dframe['preOptWeights']).astype(int) * abs(dframe['preOptWeights'].fillna(0.0)) + dframe['shortBound'].fillna(0.0) + rebalConfig.holdingThreshold / 100.0) dframe = dframe.assign(shortBoundHard=dframe[['shortBoundHard', 'shortBound']].min(axis=1)) """ ## reflect exchange holidays if rebalConfig.applyExchangeHolidays and (not optimal): lstHolidays = cls.getExchangeHolidays(tradeDate) dframe = dframe.assign(holiday=dframe['modelCountry'].isin(lstHolidays)) dframe.loc[dframe['holiday'], 'longBoundHard'] \ = Filter.bound(dframe.loc[dframe['holiday'], 'preOptWeights'].fillna(0.0), lower=0.0) \ + rebalConfig.holdingThreshold / 100.0 dframe.loc[dframe['holiday'], 'shortBoundHard'] \ = Filter.bound(-dframe.loc[dframe['holiday'], 'preOptWeights'].fillna(0.0), lower=0.0) \ + rebalConfig.holdingThreshold / 100.0 ## reflect exchange holidays """ if 'holiday' not in dframe.columns: dframe = dframe.assign(holiday=False) ## NEW reflect exchange holidays if rebalConfig.applyExchangeHolidays and (not optimal): try: lstHolidays = cls.getExchangeHolidays(tradeDate) PyLog.info(f"tradeDate = {tradeDate}, lstHolidays={lstHolidays}") dframe = dframe.assign(holiday=dframe['modelCountry'].isin(lstHolidays)) cur = dframe['preOptWeights'].fillna(0.0) cur_long = Filter.bound(cur, lower=0.0) cur_short = Filter.bound(-cur, lower=0.0) dframe.loc[dframe['holiday'], 'longBoundHard'] = cur_long.loc[dframe['holiday']] dframe.loc[dframe['holiday'], 'shortBoundHard'] = cur_short.loc[dframe['holiday']] except Exception as e: import traceback PyLog.error(f"Exception inside holiday block: {type(e).__name__}: {e}") PyLog.error(traceback.format_exc()) raise ## NEW reflect exchange holidays ## NEW reflect exchange holidays in dfAvail also dframe = dframe.assign(idx = list(range(len(dframe)))) dfAvail = dfAvail.merge(dframe[['assetKey', 'idx']], how='left', on='assetKey') if rebalConfig.applyExchangeHolidays and (not optimal): lk = dframe.set_index('assetKey') # update only the holiday rows in dfAvail hol_keys = set(dframe.loc[dframe['holiday'], 'assetKey']) mask_av_hol = dfAvail['assetKey'].isin(hol_keys) if 'longBoundHard' in dframe.columns: dfAvail.loc[mask_av_hol, 'longBoundHard'] = dfAvail.loc[mask_av_hol, 'assetKey'].map(lk['longBoundHard']) if 'shortBoundHard' in dframe.columns: dfAvail.loc[mask_av_hol, 'shortBoundHard'] = dfAvail.loc[mask_av_hol, 'assetKey'].map(lk['shortBoundHard']) PyLog.info(f"Unique modelCountry values: {dframe['modelCountry'].unique()[:10]}") PyLog.info(f"lstHolidays: {lstHolidays}") # Debugging: verify holiday logic and propagation num_holidays = dframe['holiday'].sum() if 'holiday' in dframe.columns else 0 PyLog.info(f"Detected {num_holidays} holiday assets out of {len(dframe)}") if num_holidays > 0: PyLog.info("Sample holiday-pinned assets (up to 5 per country):") holidays_df = dframe.loc[dframe['holiday']] for ctry, sub in holidays_df.groupby('modelCountry'): sample = sub[['assetKey']].head(5) PyLog.info(f"\nCountry: {ctry} — {len(sub)} total holiday assets, showing up to 5 assetKeys:\n" + sample.to_string(index=False)) # Cross-check dfAvail bounds match for those same assets merged_check = dfAvail.merge( dframe.loc[dframe['holiday'], ['assetKey', 'longBoundHard', 'shortBoundHard']], how='inner', on='assetKey', suffixes=('_avail', '_dframe') ) diff_long = (merged_check['longBoundHard_avail'] - merged_check['longBoundHard_dframe']).abs().sum() diff_short = (merged_check['shortBoundHard_avail'] - merged_check['shortBoundHard_dframe']).abs().sum() PyLog.info(f"Holiday bounds sync check — long diff: {diff_long:.3e}, short diff: {diff_short:.3e}") ## NEW reflect exchange holidays in dfAvail also availAggr = np.zeros((len(dframe), len(dfAvail))) for n in range(len(dfAvail)): availAggr[dfAvail.iloc[n]['idx'], n] = 1 factors = rmodel['factors'] fload = rmodel['fload'].reindex(index=assets, columns=factors) mu = dframe[['alpha']].to_numpy() D = np.diag(rmodel['srisk'].reindex(index=assets) ** 2) """ # === GQR specret-based specific covariance (our own) === gqr_dir = "/data/gqr_specret_cov" date_str = tradeDate.strftime("%Y%m%d") var_path = f"{gqr_dir}/gqr_specific_var.{date_str}.csv" cov_path = f"{gqr_dir}/gqr_specific_cov.{date_str}.csv" scov_path = f"{gqr_dir}/gqr_scov_sparse.{date_str}.csv" if os.path.exists(var_path): PyLog.info(f"GQR specret: loading from disk for {date_str}") spec_var_gqr = ( pd.read_csv(var_path) .set_index("assetKey")["spec_var"] ) if os.path.exists(cov_path): cov_gqr = pd.read_csv(cov_path, index_col=0) else: cov_gqr = pd.DataFrame() if os.path.exists(scov_path): scov_gqr = pd.read_csv(scov_path) else: scov_gqr = pd.DataFrame(columns=["xKey", "yKey", "cov"]) PyLog.info( "GQR specret (disk): spec_var n=%d, cov_shape=%s, scov_pairs=%d" % (len(spec_var_gqr), cov_gqr.shape, len(scov_gqr)) ) else: PyLog.info(f"GQR specret: not found on disk for {date_str} → using model diagonal") spec_var_gqr = None cov_gqr = None scov_gqr = None # === OPTION 1: replace diagonal with our own GQR specific variance === if spec_var_gqr is not None and len(spec_var_gqr) > 0: PyLog.info("GQR specret: using custom specific variance for diagonal") D = np.diag( spec_var_gqr .reindex(index=assets) .fillna(rmodel['srisk'].reindex(index=assets) ** 2) .values ) # --- DEBUG: compare GQR variance vs model variance --- compare = pd.DataFrame({ "old_var": (rmodel['srisk'] ** 2).reindex(assets), "gqr_var": spec_var_gqr.reindex(assets), }) compare["ratio"] = compare["gqr_var"] / compare["old_var"] PyLog.info("GQR ratio stats:\n%s" % compare["ratio"].describe()) # Optional: dump to csv for inspection compare.to_csv(f"{gqr_dir}/gqr_vs_model_var.{date_str}.csv") else: PyLog.info("GQR specret: no data found, keeping original diagonal") """ """ # === Option 2 GQR specret: diagonal + sparse off-diagonal === if scov_gqr is not None and not scov_gqr.empty: PyLog.info("GQR specret: applying sparse off-diagonal covariance") key_to_idx = {k: i for i, k in enumerate(assets)} applied = 0 for _, row in scov_gqr.iterrows(): x = row["xKey"] y = row["yKey"] if x in key_to_idx and y in key_to_idx: i = key_to_idx[x] j = key_to_idx[y] c = row["cov"] D[i, j] = c D[j, i] = c applied += 1 PyLog.info(f"GQR specret: applied {applied} off-diagonal pairs to D") else: PyLog.info("GQR specret: no sparse covariance found, keeping model correlations") """ """ # --- Enforce symmetry and diagonal dominance so D is PSD for cvxpy --- try: # Symmetrize numerically D = 0.5 * (D + D.T) # Make D strictly diagonally dominant: diag > sum |off-diag| diag = np.diag(D).copy() off_sum = np.sum(np.abs(D), axis=1) - np.abs(diag) # Where diagonal is too small, bump it just above row off-diagonal sum bump_idx = diag <= off_sum n_bump = int(bump_idx.sum()) if n_bump > 0: new_diag = diag new_diag[bump_idx] = off_sum[bump_idx] + 1e-6 np.fill_diagonal(D, new_diag) PyLog.info( f"GQR specret: adjusted {n_bump} diagonal entries " f"to enforce PSD via diagonal dominance" ) else: PyLog.info("GQR specret: D already diagonally dominant; no PSD adjustment needed") except Exception as e: PyLog.info(f"GQR specret: PSD enforcement on D failed ({e}); using raw D") """ """ #Directly call our internal gqr specret function. This will take lot of time # becasue we are genreating covariances for daily sims with lookback of 1 year try: spec_var_gqr, cov_gqr, scov_gqr = process_gqr_specret( trade_date=tradeDate, lookback_days=365, # ~1 year min_obs_frac=0.5, # asset must have >= 50% of weeks annualize_weeks=52.0, # weekly residuals max_neighbors=20, # top neighbors per name min_abs_corr=0.3, # correlation threshold signal_name="specretw" ) PyLog.info( "GQR specret: spec_var_gqr n=%d, cov_gqr shape=%s, scov_gqr n_pairs=%d" % (len(spec_var_gqr), cov_gqr.shape, len(scov_gqr)) ) #Option 1 (for later): replace diagonal with our own spec_var_gqr D = np.diag(spec_var_gqr.reindex(index=assets).fillna(rmodel['srisk'].reindex(index=assets) ** 2)) # Option 2 (for later): use scov_gqr as off-diagonal structure instead of rmodel['scov'] # rmodel['scov'] = scov_gqr except Exception as e: PyLog.info("GQR specret: process_gqr_specret failed, keeping original D only. Reason: %s" % e) """ """ # === Axioma CCSC: try to replace specific covariance; fall back on local D if False/None === try: # Build a minimal dframe for join: needs assetKey and ric if 'ric' in dframe.columns: # Use the RIC already present in dframe dframe_for_join = dframe[['assetKey', 'ric']] else: # Try to get RICs from the model universe frame try: mfm = SignalMgr.getStatic('model_universe_frame') mfm = mfm.rename(columns={'RkdDisplayRIC': 'ric'}) dframe_for_join = dframe[['assetKey']].merge( mfm[['assetKey', 'ric']], how='left', on='assetKey' ) except Exception as e: PyLog.info( f"Axioma integration: unable to build RICs from model_universe_frame " f"({e}); using empty ric column." ) dframe_for_join = dframe.assign(ric=None)[['assetKey', 'ric']] # existing specific covariance in optimizer order spec_diag_vec = rmodel['srisk'].reindex(index=assets) ** 2 # this is a Series indexed by assetKey axioma_out = process_axioma_specific_cov( data_dir=AXIOMA_DATA_DIR, trade_date=tradeDate, assets=assets, dframe=dframe_for_join, spec_diag_df=spec_diag_vec, splitLongShort=True ) if axioma_out is False: PyLog.info("Axioma integration: function returned False — using local specific matrix D.") elif isinstance(axioma_out, dict) and axioma_out.get('new_spec_cov') is not None: # Use only the diagonal from Axioma hybrid matrix to keep problem light D_full = axioma_out['new_spec_cov'] D_diag = np.diag(np.diag(D_full)) # strip off-diags before giving to cvxpy D = D_diag PyLog.info(f"Axioma hybrid diag-only specific covariance applied. Shape={D.shape}") else: PyLog.info("Axioma integration: no covariance provided — using local specific matrix D.") except Exception as e: PyLog.info(f"Axioma integration failed (using local D). Reason: {e}") # === end Axioma CCSC integration === """ """ ## optional off-diagonal risk if rebalConfig.offDiagRisk: scov = rmodel['scov'] scov = scov.merge(dframe[['assetKey', 'idx']].rename(columns={'assetKey': 'xKey', 'idx': 'xIdx'}), how='inner', on='xKey') scov = scov.merge(dframe[['assetKey', 'idx']].rename(columns={'assetKey': 'yKey', 'idx': 'yIdx'}), how='inner', on='yKey') for xIdx, yIdx, cov in zip(scov['xIdx'], scov['yIdx'], scov['cov']): D[xIdx, yIdx] = D[yIdx, xIdx] = cov ## optional off-diagonal risk """ S = rmodel['fcov'].reindex(index=factors, columns=factors).to_numpy() if rebalConfig.shrinkfcov: S = matrix_shrink(S) else: PyLog.info(f"rebalConfig.shrinkfcov:{rebalConfig.shrinkfcov} using fcov S as is") F = fload.to_numpy() w = cp.Variable((len(dfAvail), 1)) aggrw = availAggr @ w wlong = cp.pos(w) wshort = cp.neg(w) # China Connect short-side stability penalty (only if file exists and scores > 4) if rebalConfig.enable_gs_stability_score == True: PyLog.info(f"Looking at GS stability files for tradeDate = {tradeDate}") china_short_penalty = build_china_connect_short_penalty(dfAvail, wshort, tradeDate) else: PyLog.info(f"GS Stability flag set to {rebalConfig.enable_gs_stability_score} for tradeDate = {tradeDate}") f = F.T @ aggrw rtau = copy.copy(rebalConfig.tau) otau = tmult * rtau atau = cp.Parameter(nonneg=True) atau.value = max(0, otau - 1) btau = cp.Parameter(nonneg=True) btau.value = min(1, otau) Lmax = cp.Parameter() Lmax.value = rebalConfig.maxLeverage ## Lmax = rebalConfig.maxLeverage ## long financing cost / short borrow cost fcost = dfAvail['fcost'].to_numpy() @ wlong bcost = dfAvail['bcost'].to_numpy() @ wshort ## expected return / risk netReturn = mu.T @ aggrw - fcost - bcost variance = cp.quad_form(f, S) + cp.quad_form(aggrw, D) ## transactions cost tmodel = tmodel.reindex(index=assets) trade = (aggrw - preOptWeights.reindex(index=assets).fillna(0.0).to_frame().to_numpy()) tbuy = cp.pos(trade) tsell = cp.neg(trade) linearCost = tmodel['linearBuyCoeff'].to_numpy() @ tbuy + tmodel['linearSellCoeff'].to_numpy() @ tsell impactBuy = cp.multiply(cp.sqrt(NAV) * rebalConfig.impactCostScale, tmodel['impactCoeffTH'].to_numpy() @ cp.power(tbuy, 3 / 2)) impactSell = cp.multiply(cp.sqrt(NAV) * rebalConfig.impactCostScale, tmodel['impactCoeffTH'].to_numpy() @ cp.power(tsell, 3 / 2)) tcost = linearCost + impactBuy + impactSell avgLinearCost = tmodel['linearAverageCoeff'].to_numpy() @ (tbuy + tsell) avgTcost = avgLinearCost + impactBuy + impactSell ## constraints ------------------------------------------------------------------------------------ constraints = [] penalty = 0 ## Freeze holiday assets: no trades on exchange holidays if rebalConfig.applyExchangeHolidays and (not optimal): try: if 'holiday' in dframe.columns and dframe['holiday'].any(): # preOptWeights in dframe order cur_vec = dframe['preOptWeights'].fillna(0.0).to_numpy().reshape(-1, 1) # indices in aggrw corresponding to holiday assets hol_idx = dframe.loc[dframe['holiday'], 'idx'].to_numpy() if len(hol_idx) > 0: # aggrw[hol_idx] == current weights → zero trades for those names constraints.append(aggrw[hol_idx] == cur_vec[hol_idx]) PyLog.info(f"Holiday freeze: pinned {len(hol_idx)} assets to preOptWeights.") except Exception as e: import traceback PyLog.error(f"Exception inside holiday-freeze block: {type(e).__name__}: {e}") PyLog.error(traceback.format_exc()) raise # Start with China Connect short penalty if present if rebalConfig.enable_gs_stability_score == True: PyLog.info("Adding china short penalty") if china_short_penalty is not None: penalty = penalty + china_short_penalty ## max leverage # constraints.append(cp.norm(w, 1) <= Lmax) # pen = cp.pos(cp.norm(w, 1) - Lmax - 0.1) pen = cp.pos(cp.norm(w, 1) - Lmax) penalty = penalty + pen ## min leverage # if rebalConfig.minLeverage is not None: # Lmin = cp.Parameter() # Lmin.value = rebalConfig.minLeverage # constraints.append(cp.norm(w, 1) >= Lmin) ## risk bound # constraints.append(variance <= (rebalConfig.maxRisk ** 2)) pen = 5 * cp.pos(variance - (rebalConfig.maxRisk ** 2)) penalty = penalty + pen ## net exposure bounds # constraints.append(cp.sum(w) >= rebalConfig.minNetExposure) # constraints.append(cp.sum(w) <= rebalConfig.maxNetExposure) pen = cp.pos(cp.sum(aggrw) - rebalConfig.maxNetExposure) \ + cp.pos(rebalConfig.minNetExposure - cp.sum(aggrw)) penalty = penalty + pen ## individual position bounds - hard --------------------------------------------------------- constraints.append(cp.max(availAggr @ wlong - dframe[['longBoundHard']].to_numpy()) <= 0.0) constraints.append(cp.max(availAggr @ wshort - dframe[['shortBoundHard']].to_numpy()) <= 0.0) constraints.append(cp.max(wlong - dfAvail[['longBoundHard']].to_numpy()) <= 0.0) constraints.append(cp.max(wshort - dfAvail[['shortBoundHard']].to_numpy()) <= 0.0) ## individual position bounds - soft --------------------------------------------------------- pen = cp.sum(cp.pos(cp.pos(aggrw) - dframe[['longBoundSoft']].to_numpy())) \ + cp.sum(cp.pos(cp.neg(aggrw) - dframe[['shortBoundSoft']].to_numpy())) penalty = penalty + pen ## apply combined position bounds =========================================================== if rebalConfig.offDiagRisk: # if rebalConfig.applyExchangeHolidays and not 'HK' in lstHolidays: combAggr = np.zeros((len(scov), len(assets))) for n in range(len(scov)): combAggr[n, scov['xIdx'].iloc[n]] = combAggr[n, scov['yIdx'].iloc[n]] = 1 dfm = scov.merge(dframe[['assetKey', 'longBoundSoft', 'shortBoundSoft']]. rename( columns={'assetKey': 'xKey', 'longBoundSoft': 'xLong', 'shortBoundSoft': 'xShort'}), how='left', on='xKey') dfm = dfm.merge(dframe[['assetKey', 'longBoundSoft', 'shortBoundSoft']]. rename( columns={'assetKey': 'yKey', 'longBoundSoft': 'yLong', 'shortBoundSoft': 'yShort'}), how='left', on='yKey') dfm = dfm.assign(longBoundSoft=1.05 * dfm[['xLong', 'yLong']].max(axis=1)) dfm = dfm.assign(shortBoundSoft=1.05 * dfm[['xShort', 'yShort']].max(axis=1)) # dfm = dfm.assign(grossBound = 1.5 * dfm[['xLong', 'yLong', 'xShort', 'yShort']].max(axis=1)) if len(dfm) > 0: # constraints.append(cp.max(cp.pos(aggr @ w) - dfm[['longBound']].to_numpy()) <= 0.0) # constraints.append(cp.max(cp.neg(aggr @ w) - dfm[['shortBound']].to_numpy()) <= 0.0) pen = cp.sum(cp.pos(cp.pos(combAggr @ aggrw) - dfm[['longBoundSoft']].to_numpy())) \ + cp.sum(cp.pos(cp.neg(combAggr @ aggrw) - dfm[['shortBoundSoft']].to_numpy())) penalty = penalty + pen ## ========================================================================================== ## country net/gross exposures if not Strategy.isSingleCountry(stratName): cBounds = SignalMgr.get(rebalConfig.countryBoundName, signalDate, stratName) # if rebalConfig.applyExchangeHolidays: # cBounds = cBounds[~cBounds['cfactor'].isin(['country_' + x for x in lstHolidays])] cfactors = [x for x in factors if x.startswith('country_') and x in cBounds['cfactor'].tolist()] cBounds = cBounds.set_index('cfactor') cBounds = cBounds.reindex(index=cfactors) cBounds = cBounds.assign(maxGross=cBounds['maxGross'].fillna(0.0)) cBounds = cBounds.assign(minNet=cBounds['minNet'].fillna(0.0)) cBounds = cBounds.assign(maxNet=cBounds['maxNet'].fillna(0.0)) if rebalConfig.tightCountry: PyLog.info(f"tightening country bounds rebalConfig.tightCountry: {rebalConfig.tightCountry} stratName:{stratName}") cBounds['minNet'] = -0.0 cBounds['maxNet'] = 0.0 #PyLog.info(f"these cBounds are\n{cBounds} total max gross {cBounds.maxGross.sum() :.4f} ") # china hack. lower china leave room for JP, TH cBounds.loc["country_CN", "maxGross"] = 0.34 cBounds.loc["country_JP", "maxGross"] = 0.40 cBounds.loc["country_TH", "maxGross"] = 0.16 #PyLog.info(f"after setting china these cBounds are\n{cBounds} total max gross {cBounds.maxGross.sum() :.4f} ") cNetExp = fload.reindex(index=assets, columns=cfactors).to_numpy().T @ aggrw maxNet = cBounds[['maxNet']].to_numpy().reshape(cNetExp.shape) minNet = cBounds[['minNet']].to_numpy().reshape(cNetExp.shape) #constraints.append(cp.max(cNetExp - maxNet) <= 0.0) #constraints.append(cp.min(cNetExp - minNet) >= 0.0) if rebalConfig.tightCountry: K = 50 PyLog.info(f"tightening country bounds with high penalty {K} rebalConfig.tightCountry: {rebalConfig.tightCountry} stratName:{stratName}") pen = K * cp.sum(cp.pos(cNetExp - maxNet)) + K * cp.sum(cp.pos(minNet - cNetExp)) else: pen = cp.sum(cp.pos(cNetExp - maxNet)) + cp.sum(cp.pos(minNet - cNetExp)) penalty = penalty + pen cGrossExp = fload.reindex(index=assets, columns=cfactors).to_numpy().T @ cp.abs(aggrw) # maxGross = (rebalConfig.maxLeverage * 1.05 * cBounds[['maxGross']]).to_numpy().reshape(cGrossExp.shape) maxGross = (rebalConfig.maxLeverage * cBounds[['maxGross']]).to_numpy().reshape(cGrossExp.shape) # constraints.append(cp.max(cGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(cGrossExp - maxGross)) penalty = penalty + pen ## industry net/gross exposures iBounds = SignalMgr.get(rebalConfig.industryBoundName, signalDate, stratName) #PyLog.info(f"iBounds are: {iBounds}") ifactors = [x for x in factors if x.startswith('ind_') if x in iBounds['ifactor'].tolist()] iBounds = iBounds.set_index('ifactor') iBounds = iBounds.reindex(index=ifactors) iBounds = iBounds.assign(maxGross=iBounds['maxGross'].fillna(0.0)) iBounds = iBounds.assign(minNet=iBounds['minNet'].fillna(0.0)) iBounds = iBounds.assign(maxNet=iBounds['maxNet'].fillna(0.0)) iNetExp = fload.reindex(index=assets, columns=ifactors).to_numpy().T @ aggrw maxNet = iBounds[['maxNet']].to_numpy().reshape(iNetExp.shape) minNet = iBounds[['minNet']].to_numpy().reshape(iNetExp.shape) # constraints.append(cp.max(iNetExp - maxNet) <= 0.0) # constraints.append(cp.min(iNetExp - minNet) >= 0.0) pen = cp.sum(cp.pos(iNetExp - maxNet)) + cp.sum(cp.pos(minNet - iNetExp)) penalty = penalty + pen iGrossExp = fload.reindex(index=assets, columns=ifactors).to_numpy().T @ cp.abs(aggrw) # maxGross = (rebalConfig.maxLeverage * 1.05 * iBounds[['maxGross']]).to_numpy().reshape(iGrossExp.shape) maxGross = (rebalConfig.maxLeverage * iBounds[['maxGross']]).to_numpy().reshape(iGrossExp.shape) # constraints.append(cp.max(iGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(iGrossExp - maxGross)) penalty = penalty + pen ## size net/gross exposures sfactors = [x for x in factors if x.startswith('size_')] sBounds = SignalMgr.get(rebalConfig.sizeBoundName, signalDate, stratName).set_index('sfactor') sBounds = sBounds.reindex(index=sfactors) sBounds = sBounds.assign(maxGross=sBounds['maxGross'].fillna(0.0)) sBounds = sBounds.assign(minNet=sBounds['minNet'].fillna(0.0)) sBounds = sBounds.assign(maxNet=sBounds['maxNet'].fillna(0.0)) sNetExp = fload.reindex(index=assets, columns=sfactors).to_numpy().T @ aggrw maxNet = sBounds[['maxNet']].to_numpy().reshape(sNetExp.shape) minNet = sBounds[['minNet']].to_numpy().reshape(sNetExp.shape) # constraints.append(cp.max(sNetExp - maxNet) <= 0.0) # constraints.append(cp.min(sNetExp - minNet) >= 0.0) pen = cp.sum(cp.pos(sNetExp - maxNet)) + cp.sum(cp.pos(minNet - sNetExp)) penalty = penalty + pen sGrossExp = fload.reindex(index=assets, columns=sfactors).to_numpy().T @ cp.abs(aggrw) # maxGross = (rebalConfig.maxLeverage * 1.05 * sBounds[['maxGross']]).to_numpy().reshape(sGrossExp.shape) maxGross = (rebalConfig.maxLeverage * sBounds[['maxGross']]).to_numpy().reshape(sGrossExp.shape) sbounds_maxGross = sBounds[['maxGross']] PyLog.info(f"maxGross: {maxGross} = rebalConfig.maxLeverage {rebalConfig.maxLeverage} * {sbounds_maxGross}") # constraints.append(cp.max(sGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(sGrossExp - maxGross)) penalty = penalty + pen ## net exposure to sbeta_market mNetExp = fload.reindex(index=assets, columns=['sbeta_market']).to_numpy().T @ aggrw constraints.append(cp.min(mNetExp) >= rebalConfig.minBetaAdjNet) constraints.append(cp.max(mNetExp) <= rebalConfig.maxBetaAdjNet) PyLog.info(f"mNetExp: {mNetExp} rebalConfig.minBetaAdjNet:{rebalConfig.minBetaAdjNet} rebalConfig.maxBetaAdjNet:{rebalConfig.maxBetaAdjNet}") ## phase out value_liq in 2017~ ============================================================ # if PyDate.ge(signalDate, 20170101) and PyDate.le(signalDate, PyMonth.lastWeekday(202012)): # # if PyDate.ge(signalDate, 20170101): # vliqExp = fload.reindex(index=assets, columns=['value_liq']).to_numpy().T @ w # bound = 2.0 * Filter.bound(PyDate.span(signalDate, 20170331) / 90, lower=0.01) # constraints.append(cp.max(vliqExp) <= bound) # constraints.append(cp.min(vliqExp) >= -bound) ## ========================================================================================== # mfm = SignalMgr.getStatic('model_universe_frame') mfm = mfm.rename(columns={'RkdTicker': 'ticker', 'RkdDisplayRIC': 'ric', 'Country': 'quoteCountry'}) dframe = mfm[['assetKey', 'ticker', 'quoteCountry', 'ric']].merge(dframe, how='right', on='assetKey') df_lbs_sbs_ne0 = dframe[ (dframe['longBoundSoft'] !=0 ) | (dframe['shortBoundSoft'] !=0 ) ] univ_wt0 = dframe[ dframe['univwt']!=0 ] preOptWeights0 = dframe[ (dframe['preOptWeights']==0) | ( dframe['preOptWeights'].isnull() ) ] sb_lb_0 = dframe[ (dframe['longBoundHard'] !=0 ) | (dframe['shortBoundHard'] !=0 ) ] # sb_lb_0 = dframe user = os.environ['USER'] sb_lb_0.to_csv(f"/home/{user}/scratch/nonZeroHardBounds.csv", index=False, columns=['assetKey','ticker','quoteCountry','ric']) dframe.to_csv(f"/home/{user}/scratch/preOptUniv.csv", index=False, columns=['assetKey','ticker','quoteCountry','ric']) ## apply alpha theme exposure bounds ======================================================== if rebalConfig.themeBoundName is not None: themeBounds = SignalMgr.get(rebalConfig.themeBoundName, signalDate, stratName) themeBounds = FrameUtil.toSeries(themeBounds, keyCol='factor', valCol='bound') * rebalConfig.maxLeverage themes = list(themeBounds.index) themeExp = fload.reindex(index=assets, columns=themes).to_numpy().T @ aggrw # constraints.append(cp.max(themeExp - themeBounds.to_numpy().reshape(themeExp.shape)) <= 0) # constraints.append(cp.min(themeExp + themeBounds.to_numpy().reshape(themeExp.shape)) >= 0) pen = cp.sum(cp.pos(themeExp - themeBounds.to_numpy().reshape(themeExp.shape))) \ + cp.sum(cp.pos(- themeExp - themeBounds.to_numpy().reshape(themeExp.shape))) penalty = penalty + pen ## here we are constructing symmetric bounds around 0 but it is the upper bound that we are ## primarily concerned about ## ========================================================================================== ## penalty - apply max trade constraints ==================================================== if (not optimal) and (rebalConfig.maxAdvProp is not None): dfm = SignalMgr.getFrame(rebalConfig.advName, signalDate, stratName) dfm = dframe[['assetKey']].merge(dfm[['assetKey', rebalConfig.advName]], how='left', on='assetKey') dfm = dfm.assign( maxTradeWeights=rebalConfig.maxAdvProp * dfm[rebalConfig.advName].fillna(0.0) * 1000000 / NAV) ## may need to relax this condition for certain conditions (i.e. forced trading) # constraints.append(cp.max(cp.abs(trade) - dfm[['maxTradeWeights']].to_numpy()) <= 0.0) pen = (cp.abs(trade) - dfm[['maxTradeWeights']].to_numpy()) / dfm[['maxTradeWeights']].to_numpy() # pen = 4 * cp.max(cp.pos(pen)) pen = 4 * cp.sum(cp.pos(pen)) penalty = penalty + pen ## ========================================================================================== ################ THIS ENTIRE SECTION IS ADDING TRBC ACTIVIT LEVEL PENALTIES BASED ON SPECRET COV import model.utils.ric_tracer as rictr dframe = rictr.classify_rics_by_trbc(dframe) # === TRBC activity + correlation-based cluster penalty (Path B) === try: trbc_lambda = getattr(rebalConfig, "trbcClusterPenaltyLambda", 0.01) trbc_pen = build_trbc_cluster_penalty( dframe=dframe, assets=assets, aggrw=aggrw, tradeDate=tradeDate, gqr_dir="/data/gqr_specret_cov", corr_threshold=0.5, min_cluster_size=3, lambda_cluster=trbc_lambda, ) if isinstance(trbc_pen, cp.Expression): penalty = penalty + trbc_pen PyLog.info("TRBC cluster penalty: added to objective") else: PyLog.info("TRBC cluster penalty: nothing to add (no clusters / files missing)") except Exception as e: PyLog.info(f"TRBC cluster penalty: failed to build ({type(e).__name__}: {e}) → skipping") ################ THIS ENTIRE SECTION IS ADDING TRBC ACTIVIT LEVEL PENALTIES BASED ON SPECRET COV gamma = cp.Parameter(nonneg=True) # Uncomment this to print final dataframe before optimization starts #dframe.to_csv("dframe_final.csv") PyLog.info(f"constraints==== {len(constraints)}===") if rebalConfig.adaptiveGamma: prevGamma = 0.0 currGamma = 0.0 prevRisk = 0.0 currRisk = 0.0 offset = 1.0 while Real.isZero(currRisk) or \ abs(currRisk - rebalConfig.adaptiveGammaTargetRisk) > rebalConfig.adaptiveGammaTolerance: if Real.isZero(currRisk): rgamma = rebalConfig.gamma elif Real.isZero(prevRisk): rgamma = currGamma + np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) * offset elif Real.isPositive(np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) * np.sign(prevRisk - rebalConfig.adaptiveGammaTargetRisk)): rgamma = currGamma + np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) else: rgamma = (prevGamma + currGamma) / 2 offset = offset / 4 gamma.value = gmult * rgamma ## optimization setup if optimal: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - penalty), constraints) else: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - btau * tcost - atau * avgTcost - penalty), constraints) numIters = cls.solveProblem(rebalConfig=rebalConfig, problem=problem, maxIter=maxIter, cycleIter=cycleIter, solver=solver, verbose=verbose, checkDCP=checkDCP, optimal=optimal) if w.value is None: PyLog.info(" Optimizer failed to converge") w.value = dfAvail[['preOptWeights']].fillna(0.0).to_numpy() weights = pd.Series(aggrw.value.flatten(), index=assets) prevGamma = currGamma prevRisk = currRisk currGamma = rgamma currRisk = RiskModel.computeRisk(rmodel, weights) else: rgamma = rebalConfig.gamma gamma.value = gmult * rgamma ## optimization setup if optimal: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - penalty), constraints) else: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - btau * tcost - atau * avgTcost - penalty), constraints) ## problem.solve numIters = cls.solveProblem(rebalConfig=rebalConfig, problem=problem, maxIter=maxIter, cycleIter=cycleIter, solver=solver, verbose=verbose, checkDCP=checkDCP, optimal=optimal) if w.value is None: PyLog.info(" Optimizer failed to converge") w.value = dfAvail[['preOptWeights']].fillna(0.0).to_numpy() # Log TRBC cluster exposures and implied penalty log_trbc_cluster_exposures( tradeDate, assets, aggrw, gqr_dir="/data/gqr_specret_cov", lambda_cluster=trbc_lambda) dfAvail = dfAvail.assign(optimalWeights = list(w.value.flatten())) dfAvail = dfAvail.assign(postOptWeights = dfAvail['optimalWeights']) # dfAvail.loc[abs(dfAvail['postOptWeights']) < rebalConfig.holdingThreshold, 'postOptWeights'] = 0.0 dframe = dframe.assign(optimalWeights = availAggr @ dfAvail['optimalWeights']) dframe = dframe.assign(postOptWeights = availAggr @ dfAvail['postOptWeights']) dfWeights = dfWeights.merge(dframe[['assetKey', 'optimalWeights', 'postOptWeights']], how='outer', on='assetKey') dfWeights.loc[abs(dfWeights['postOptWeights'].fillna(0.0)) < rebalConfig.holdingThreshold, 'postOptWeights'] = 0.0 dfWeights = dfWeights.assign( tradeWeights = dfWeights['postOptWeights'].fillna(0.0) - dfWeights['preOptWeights'].fillna(0.0)) # --- Start China penalty post-solve logging --- if rebalConfig.enable_gs_stability_score == True: if china_short_penalty is not None and hasattr(china_short_penalty, "_china_penalty_info"): try: info = china_short_penalty._china_penalty_info weights_arr = china_short_penalty._china_penalty_weights scale = china_short_penalty._china_scale PyLog.info("ChinaConnect: post-solve penalty details for penalized assets (top 50):") for idx, assetKey, score, pw in info[:50]: if wshort.value is None: short_w = None contrib = None else: short_w = float(wshort.value[idx]) contrib = scale * pw * short_w if short_w is not None else None PyLog.info( f" assetKey={assetKey}, score={score}, penaltyWeight={pw}, " f"shortWeight={(short_w if short_w is not None else 'None')}, " f"penaltyContribution={(contrib if contrib is not None else 'None')}" ) if len(info) > 50: PyLog.info(f" ...and {len(info) - 50} more penalized assets") total_pen = float(china_short_penalty.value) if china_short_penalty.value is not None else None PyLog.info(f"ChinaConnect: total China penalty = {total_pen}") except Exception as e: PyLog.error(f"ChinaConnect: error computing post-solve penalty log: {e}") # --- End China penalty post-solve logging --- # weights = pd.Series(w.value.flatten(), index=assets) else: PyLog.info(f"Entering split block. rebalConfig.splitLongShort:{rebalConfig.splitLongShort} rebalConfig.splitLongShortStartDate:{rebalConfig.splitLongShortStartDate}") PyLog.info(f"rebalConfig.applyExchangeHolidays:{rebalConfig.applyExchangeHolidays} optimal:{optimal}") """ ## reflect exchange holidays ------------------------------------------------------------------- if rebalConfig.applyExchangeHolidays and (not optimal): lstHolidays = cls.getExchangeHolidays(tradeDate) dframe = dframe.assign(holiday=dframe['modelCountry'].isin(lstHolidays)) dframe.loc[dframe['holiday'], 'longBoundHard'] \ = Filter.bound(dframe.loc[dframe['holiday'], 'preOptWeights'].fillna(0.0), lower=0.0) \ + rebalConfig.holdingThreshold / 100.0 dframe.loc[dframe['holiday'], 'shortBoundHard'] \ = Filter.bound(-dframe.loc[dframe['holiday'], 'preOptWeights'].fillna(0.0), lower=0.0) \ + rebalConfig.holdingThreshold / 100.0 ## --------------------------------------------------------------------------------------------- """ ## NEW reflect exchange holidays if rebalConfig.applyExchangeHolidays and (not optimal): lstHolidays = cls.getExchangeHolidays(tradeDate) dframe = dframe.assign(holiday=dframe['modelCountry'].isin(lstHolidays)) cur = dframe['preOptWeights'].fillna(0.0) cur_long = Filter.bound(cur, lower=0.0) cur_short = Filter.bound(-cur, lower=0.0) dframe.loc[dframe['holiday'], 'longBoundHard'] = cur_long.loc[dframe['holiday']] dframe.loc[dframe['holiday'], 'shortBoundHard'] = cur_short.loc[dframe['holiday']] factors = rmodel['factors'] fload = rmodel['fload'].reindex(index=assets, columns=factors) ## NEW reflect exchange holidays mu = dframe[['alpha']].to_numpy() D = np.diag(rmodel['srisk'].reindex(index=assets) ** 2) ## optional off-diagonal risk ------------------------------------------------------------------------------ PyLog.info(f"rebalConfig.offDiagRisk:{rebalConfig.offDiagRisk}") if rebalConfig.offDiagRisk: scov = rmodel['scov'] dfm = pd.DataFrame({'assetKey': assets, 'idx': range(len(assets))}) scov = scov.merge(dfm.rename(columns={'assetKey': 'xKey', 'idx': 'xIdx'}), how='inner', on='xKey') scov = scov.merge(dfm.rename(columns={'assetKey': 'yKey', 'idx': 'yIdx'}), how='inner', on='yKey') for xIdx, yIdx, cov in zip(scov['xIdx'], scov['yIdx'], scov['cov']): D[xIdx, yIdx] = D[yIdx, xIdx] = cov ## --------------------------------------------------------------------------------------------------------- S = rmodel['fcov'].reindex(index=factors, columns=factors).to_numpy() if rebalConfig.shrinkfcov: S = matrix_shrink(S) else: PyLog.info(f"rebalConfig.shrinkfcov:{rebalConfig.shrinkfcov} using fcov S as is") F = fload.to_numpy() w = cp.Variable((len(assets), 1)) ## wlong = cp.maximum( w, 0) ## wshort = cp.maximum(-w, 0) wlong = cp.pos(w) wshort = cp.neg(w) f = F.T @ w rtau = rebalConfig.tau otau = tmult * rtau atau = cp.Parameter(nonneg=True) atau.value = max(0, otau - 1) btau = cp.Parameter(nonneg=True) btau.value = min(1, otau) Lmax = cp.Parameter() Lmax.value = rebalConfig.maxLeverage ## Lmax = rebalConfig.maxLeverage ## long financing cost / short borrow cost fcost = dframe['fcost'].to_numpy() @ wlong bcost = dframe['bcost'].to_numpy() @ wshort ## expected return / risk netReturn = mu.T @ w - fcost - bcost variance = cp.quad_form(f, S) + cp.quad_form(w, D) ## transactions cost tmodel = tmodel.reindex(index=assets) trade = (w - preOptWeights.reindex(index=assets).fillna(0.0).to_frame().to_numpy()) tbuy = cp.pos(trade) tsell = cp.neg(trade) linearCost = tmodel['linearBuyCoeff'].to_numpy() @ tbuy + tmodel['linearSellCoeff'].to_numpy() @ tsell ## impactCost = (tmodel['impactCoeffTH'].to_numpy() * ((NAV * (tbuy + tsell)) ** (3/2))) / NAV ## impactCost = tmodel['impactCoeffTH'].to_numpy() * cp.multiply(1/NAV, cp.power(cp.multiply(NAV, tbuy + tsell), 3/2)) ## impactBuy = cp.multiply(1 / NAV, tmodel['impactCoeffTH'].to_numpy() * cp.power(cp.multiply(NAV, tbuy), 3 / 2)) ## impactSell = cp.multiply(1 / NAV, tmodel['impactCoeffTH'].to_numpy() * cp.power(cp.multiply(NAV, tsell), 3 / 2)) impactBuy = cp.multiply(cp.sqrt(NAV) * rebalConfig.impactCostScale, tmodel['impactCoeffTH'].to_numpy() @ cp.power(tbuy, 3 / 2)) impactSell = cp.multiply(cp.sqrt(NAV) * rebalConfig.impactCostScale, tmodel['impactCoeffTH'].to_numpy() @ cp.power(tsell, 3 / 2)) tcost = linearCost + impactBuy + impactSell avgLinearCost = tmodel['linearAverageCoeff'].to_numpy() @ (tbuy + tsell) avgTcost = avgLinearCost + impactBuy + impactSell ## constraints -------------------------------------------------------------------- constraints = [] penalty = 0 ## max leverage # constraints.append(cp.norm(w, 1) <= Lmax) # pen = cp.pos(cp.norm(w, 1) - Lmax - 0.1) pen = cp.pos(cp.norm(w, 1) - Lmax) penalty = penalty + pen ## min leverage # if rebalConfig.minLeverage is not None: # Lmin = cp.Parameter() # Lmin.value = rebalConfig.minLeverage # constraints.append(cp.norm(w, 1) >= Lmin) ## risk bound # constraints.append(variance <= (rebalConfig.maxRisk ** 2)) pen = 5 * cp.pos(variance - (rebalConfig.maxRisk ** 2)) penalty = penalty + pen ## net exposure bounds # constraints.append(cp.sum(w) >= rebalConfig.minNetExposure) # constraints.append(cp.sum(w) <= rebalConfig.maxNetExposure) pen = cp.pos(cp.sum(w) - rebalConfig.maxNetExposure) \ + cp.pos(rebalConfig.minNetExposure - cp.sum(w)) penalty = penalty + pen ## individual position bounds - hard --------------------------------------------------------- constraints.append(cp.max(wlong - dframe[['longBoundHard']].to_numpy()) <= 0.0) constraints.append(cp.max(wshort - dframe[['shortBoundHard']].to_numpy()) <= 0.0) ## individual position bounds - soft --------------------------------------------------------- pen = cp.sum(cp.pos(cp.pos(w) - dframe[['longBoundSoft']].to_numpy())) \ + cp.sum(cp.pos(cp.neg(w) - dframe[['shortBoundSoft']].to_numpy())) penalty = penalty + pen ## apply combined position bounds =========================================================== if rebalConfig.offDiagRisk: # if rebalConfig.applyExchangeHolidays and not 'HK' in lstHolidays: aggr = np.zeros((len(scov), len(assets))) for n in range(len(scov)): aggr[n, scov['xIdx'].iloc[n]] = aggr[n, scov['yIdx'].iloc[n]] = 1 dfm = scov.merge(dframe[['assetKey', 'longBoundSoft', 'shortBoundSoft']]. rename(columns={'assetKey': 'xKey', 'longBoundSoft': 'xLong', 'shortBoundSoft': 'xShort'}), how='left', on='xKey') dfm = dfm.merge(dframe[['assetKey', 'longBoundSoft', 'shortBoundSoft']]. rename(columns={'assetKey': 'yKey', 'longBoundSoft': 'yLong', 'shortBoundSoft': 'yShort'}), how='left', on='yKey') dfm = dfm.assign(longBoundSoft = 1.05 * dfm[['xLong', 'yLong']].max(axis=1)) dfm = dfm.assign(shortBoundSoft = 1.05 * dfm[['xShort', 'yShort']].max(axis=1)) # dfm = dfm.assign(grossBound = 1.5 * dfm[['xLong', 'yLong', 'xShort', 'yShort']].max(axis=1)) if len(dfm) > 0: # constraints.append(cp.max(cp.pos(aggr @ w) - dfm[['longBound']].to_numpy()) <= 0.0) # constraints.append(cp.max(cp.neg(aggr @ w) - dfm[['shortBound']].to_numpy()) <= 0.0) pen = cp.sum(cp.pos(cp.pos(aggr @ w) - dfm[['longBoundSoft']].to_numpy())) \ + cp.sum(cp.pos(cp.neg(aggr @ w) - dfm[['shortBoundSoft']].to_numpy())) penalty = penalty + pen ## ========================================================================================== ## country net/gross exposures if not Strategy.isSingleCountry(stratName): cBounds = SignalMgr.get(rebalConfig.countryBoundName, signalDate, stratName) # if rebalConfig.applyExchangeHolidays: # cBounds = cBounds[~cBounds['cfactor'].isin(['country_' + x for x in lstHolidays])] cfactors = [x for x in factors if x.startswith('country_') and x in cBounds['cfactor'].tolist()] cBounds = cBounds.set_index('cfactor') cBounds = cBounds.reindex(index=cfactors) cBounds = cBounds.assign(maxGross = cBounds['maxGross'].fillna(0.0)) cBounds = cBounds.assign(minNet = cBounds['minNet'].fillna(0.0)) cBounds = cBounds.assign(maxNet = cBounds['maxNet'].fillna(0.0)) if rebalConfig.tightCountry: PyLog.info(f"tightening country bounds rebalConfig.tightCountry: {rebalConfig.tightCountry} stratName:{stratName}") cBounds['minNet'] = -0.0 cBounds['maxNet'] = 0.0 # aggrw is w cNetExp = fload.reindex(index=assets, columns=cfactors).to_numpy().T @ w maxNet = cBounds[['maxNet']].to_numpy().reshape(cNetExp.shape) minNet = cBounds[['minNet']].to_numpy().reshape(cNetExp.shape) PyLog.info(f"here cBounds are\n{cBounds}") #constraints.append(cp.max(cNetExp - maxNet) <= 0.0) #constraints.append(cp.min(cNetExp - minNet) >= 0.0) if rebalConfig.tightCountry: K = 50 PyLog.info(f"tightening country bounds with high penalty {K} rebalConfig.tightCountry: {rebalConfig.tightCountry} stratName:{stratName}") pen = K * cp.sum(cp.pos(cNetExp - maxNet)) + K * cp.sum(cp.pos(minNet - cNetExp)) else: pen = cp.sum(cp.pos(cNetExp - maxNet)) + cp.sum(cp.pos(minNet - cNetExp)) penalty = penalty + pen cGrossExp = fload.reindex(index=assets, columns=cfactors).to_numpy().T @ cp.abs(w) # maxGross = (rebalConfig.maxLeverage * 1.05 * cBounds[['maxGross']]).to_numpy().reshape(cGrossExp.shape) maxGross = (rebalConfig.maxLeverage * cBounds[['maxGross']]).to_numpy().reshape(cGrossExp.shape) # constraints.append(cp.max(cGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(cGrossExp - maxGross)) penalty = penalty + pen ## industry net/gross exposures iBounds = SignalMgr.get(rebalConfig.industryBoundName, signalDate, stratName) ifactors = [x for x in factors if x.startswith('ind_') if x in iBounds['ifactor'].tolist()] iBounds = iBounds.set_index('ifactor') iBounds = iBounds.reindex(index=ifactors) iBounds = iBounds.assign(maxGross = iBounds['maxGross'].fillna(0.0)) iBounds = iBounds.assign(minNet = iBounds['minNet'].fillna(0.0)) iBounds = iBounds.assign(maxNet = iBounds['maxNet'].fillna(0.0)) iNetExp = fload.reindex(index=assets, columns=ifactors).to_numpy().T @ w maxNet = iBounds[['maxNet']].to_numpy().reshape(iNetExp.shape) minNet = iBounds[['minNet']].to_numpy().reshape(iNetExp.shape) # constraints.append(cp.max(iNetExp - maxNet) <= 0.0) # constraints.append(cp.min(iNetExp - minNet) >= 0.0) pen = cp.sum(cp.pos(iNetExp - maxNet)) + cp.sum(cp.pos(minNet - iNetExp)) penalty = penalty + pen iGrossExp = fload.reindex(index=assets, columns=ifactors).to_numpy().T @ cp.abs(w) # maxGross = (rebalConfig.maxLeverage * 1.05 * iBounds[['maxGross']]).to_numpy().reshape(iGrossExp.shape) maxGross = (rebalConfig.maxLeverage * iBounds[['maxGross']]).to_numpy().reshape(iGrossExp.shape) # constraints.append(cp.max(iGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(iGrossExp - maxGross)) penalty = penalty + pen ## size net/gross exposures sfactors = [x for x in factors if x.startswith('size_')] sBounds = SignalMgr.get(rebalConfig.sizeBoundName, signalDate, stratName).set_index('sfactor') sBounds = sBounds.reindex(index=sfactors) sBounds = sBounds.assign(maxGross = sBounds['maxGross'].fillna(0.0)) sBounds = sBounds.assign(minNet = sBounds['minNet'].fillna(0.0)) sBounds = sBounds.assign(maxNet = sBounds['maxNet'].fillna(0.0)) sNetExp = fload.reindex(index=assets, columns=sfactors).to_numpy().T @ w maxNet = sBounds[['maxNet']].to_numpy().reshape(sNetExp.shape) minNet = sBounds[['minNet']].to_numpy().reshape(sNetExp.shape) # constraints.append(cp.max(sNetExp - maxNet) <= 0.0) # constraints.append(cp.min(sNetExp - minNet) >= 0.0) pen = cp.sum(cp.pos(sNetExp - maxNet)) + cp.sum(cp.pos(minNet - sNetExp)) penalty = penalty + pen PyLog.info(f"sfactors: {sfactors}") PyLog.info(f"fload:\n{fload[sfactors].iloc[0]}") sGrossExp = fload.reindex(index=assets, columns=sfactors).to_numpy().T @ cp.abs(w) # maxGross = (rebalConfig.maxLeverage * 1.05 * sBounds[['maxGross']]).to_numpy().reshape(sGrossExp.shape) maxGross = (rebalConfig.maxLeverage * sBounds[['maxGross']]).to_numpy().reshape(sGrossExp.shape) PyLog.info(f"sBounds are\n{sBounds}") sbounds_maxGross = sBounds[['maxGross']] PyLog.info(f"maxGross: {maxGross} = rebalConfig.maxLeverage {rebalConfig.maxLeverage} * {sbounds_maxGross}") # constraints.append(cp.max(sGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(sGrossExp - maxGross)) penalty = penalty + pen ## net exposure to sbeta_market mNetExp = fload.reindex(index=assets, columns=['sbeta_market']).to_numpy().T @ w constraints.append(cp.min(mNetExp) >= rebalConfig.minBetaAdjNet) constraints.append(cp.max(mNetExp) <= rebalConfig.maxBetaAdjNet) PyLog.info(f"mNetExp: {mNetExp} rebalConfig.minBetaAdjNet:{rebalConfig.minBetaAdjNet} rebalConfig.maxBetaAdjNet:{rebalConfig.maxBetaAdjNet}") ## phase out value_liq in 2017~ ============================================================ # if PyDate.ge(signalDate, 20170101) and PyDate.le(signalDate, PyMonth.lastWeekday(202012)): # # if PyDate.ge(signalDate, 20170101): # vliqExp = fload.reindex(index=assets, columns=['value_liq']).to_numpy().T @ w # bound = 2.0 * Filter.bound(PyDate.span(signalDate, 20170331) / 90, lower=0.01) # constraints.append(cp.max(vliqExp) <= bound) # constraints.append(cp.min(vliqExp) >= -bound) ## ========================================================================================== ## apply alpha theme exposure bounds ======================================================== if rebalConfig.themeBoundName is not None: themeBounds = SignalMgr.get(rebalConfig.themeBoundName, signalDate, stratName) themeBounds = FrameUtil.toSeries(themeBounds, keyCol='factor', valCol='bound') * rebalConfig.maxLeverage themes = list(themeBounds.index) themeExp = fload.reindex(index=assets, columns=themes).to_numpy().T @ w # constraints.append(cp.max(themeExp - themeBounds.to_numpy().reshape(themeExp.shape)) <= 0) # constraints.append(cp.min(themeExp + themeBounds.to_numpy().reshape(themeExp.shape)) >= 0) pen = cp.sum(cp.pos(themeExp - themeBounds.to_numpy().reshape(themeExp.shape))) \ + cp.sum(cp.pos(- themeExp - themeBounds.to_numpy().reshape(themeExp.shape))) penalty = penalty + pen ## here we are constructing symmetric bounds around 0 but it is the upper bound that we are ## primarily concerned about ## ========================================================================================== ## penalty - apply max trade constraints ==================================================== if (not optimal) and (rebalConfig.maxAdvProp is not None): dfm = SignalMgr.getFrame(rebalConfig.advName, signalDate, stratName) dfm = dframe[['assetKey']].merge(dfm[['assetKey', rebalConfig.advName]], how='left', on='assetKey') dfm = dfm.assign( maxTradeWeights = rebalConfig.maxAdvProp * dfm[rebalConfig.advName].fillna(0.0) * 1000000 / NAV) ## may need to relax this condition for certain conditions (i.e. forced trading) # constraints.append(cp.max(cp.abs(trade) - dfm[['maxTradeWeights']].to_numpy()) <= 0.0) pen = (cp.abs(trade) - dfm[['maxTradeWeights']].to_numpy()) / dfm[['maxTradeWeights']].to_numpy() # pen = 4 * cp.max(cp.pos(pen)) pen = 4 * cp.sum(cp.pos(pen)) penalty = penalty + pen ## ========================================================================================== gamma = cp.Parameter(nonneg=True) if rebalConfig.adaptiveGamma: prevGamma = 0.0 currGamma = 0.0 prevRisk = 0.0 currRisk = 0.0 offset = 1.0 while Real.isZero(currRisk) or \ abs(currRisk - rebalConfig.adaptiveGammaTargetRisk) > rebalConfig.adaptiveGammaTolerance: if Real.isZero(currRisk): rgamma = rebalConfig.gamma elif Real.isZero(prevRisk): rgamma = currGamma + np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) * offset elif Real.isPositive(np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) * np.sign(prevRisk - rebalConfig.adaptiveGammaTargetRisk)): rgamma = currGamma + np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) else: rgamma = (prevGamma + currGamma) / 2 offset = offset / 4 gamma.value = gmult * rgamma ## optimization setup if optimal: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - penalty), constraints) else: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - btau * tcost - atau * avgTcost - penalty), constraints) numIters = cls.solveProblem(rebalConfig=rebalConfig, problem=problem, maxIter=maxIter, cycleIter=cycleIter, solver=solver, verbose=verbose, checkDCP=checkDCP, optimal=optimal) if w.value is None: PyLog.info(" Optimizer failed to converge") w.value = dframe[['preOptWeights']].fillna(0.0).to_numpy() weights = pd.Series(w.value.flatten(), index=assets) prevGamma = currGamma prevRisk = currRisk currGamma = rgamma currRisk = RiskModel.computeRisk(rmodel, weights) else: rgamma = rebalConfig.gamma gamma.value = gmult * rgamma ## optimization setup if optimal: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - penalty), constraints) else: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - btau * tcost - atau * avgTcost - penalty), constraints) ## problem.solve numIters = cls.solveProblem(rebalConfig=rebalConfig, problem=problem, maxIter=maxIter, cycleIter=cycleIter, solver=solver, verbose=verbose, checkDCP=checkDCP, optimal=optimal) if w.value is None: PyLog.info(" Optimizer failed to converge") w.value = dframe[['preOptWeights']].fillna(0.0).to_numpy() weights = pd.Series(w.value.flatten(), index=assets) dfWeights = dfWeights.merge(weights.reset_index().rename(columns={'index': 'assetKey', 0: 'optimalWeights'}), how='outer', on='assetKey') dfWeights = dfWeights.assign(postOptWeights = dfWeights['optimalWeights']) dfWeights.loc[abs(dfWeights['postOptWeights'].fillna(0.0)) < rebalConfig.holdingThreshold, 'postOptWeights'] = 0.0 # weights.loc[abs(weights) < rebalConfig.holdingThreshold] = 0.0 # dfWeights = dfWeights.merge(weights.reset_index().rename(columns={'index': 'assetKey', 0: 'postOptWeights'}), # how='outer', on='assetKey') dctOptResult['numIters'] = numIters ## including cvxpy objects increased output file from 100K to 80MB dctOptResult['gamma'] = rgamma dctOptResult['netReturn'] = netReturn.value dctOptResult['variance'] = variance.value dfWeights = dfWeights.merge(dframe[['assetKey', 'longBoundHard', 'shortBoundHard']], how='left', on='assetKey') dfWeights = dfWeights.assign(longBoundHard = dfWeights['longBoundHard'].fillna(0.0)) dfWeights = dfWeights.assign(shortBoundHard = dfWeights['shortBoundHard'].fillna(0.0)) dfWeights = dfWeights.assign(preOptWeights = dfWeights['preOptWeights'].fillna(0.0)) dfWeights = dfWeights.assign(optimalWeights = dfWeights['optimalWeights'].fillna(0.0)) dfWeights = dfWeights.assign(postOptWeights = dfWeights['postOptWeights'].fillna(0.0)) dfWeights = dfWeights.assign(tradeWeights = dfWeights['postOptWeights'] - dfWeights['preOptWeights']) dctOptResult['dfWeights'] = dfWeights dframe = dfWeights.merge(tmodel.reset_index(), how='left', on='assetKey') linearCost = NAV * ((Filter.bound(dframe['tradeWeights'], lower=0.0) * dframe['linearBuyCoeff'].fillna(dframe['linearBuyCoeff'].max())) - (Filter.bound(dframe['tradeWeights'], upper=0.0) * dframe['linearSellCoeff'].fillna(dframe['linearSellCoeff'].max()))).sum() dctOptResult['linearCostUSD'] = linearCost impactCost = rebalConfig.impactCostScale * np.power(NAV, 3 / 2) * \ (np.power(abs(dframe['tradeWeights']), 3 / 2) * dframe['impactCoeffTH'].fillna(dframe['impactCoeffTH'].max())).sum() dctOptResult['impactCostUSD'] = impactCost dctOptResult['tcostUSD'] = linearCost + impactCost dctOptResult['fcost'] = fcost.value[0] dctOptResult['bcost'] = bcost.value[0] dctOptResult['tradeWeight'] = abs(dfWeights['tradeWeights']).sum() dctOptResult['tradeValUSD'] = dctOptResult['tradeWeight'] * dctOptResult['preOptNAV'] return dctOptResult @classmethod def getExchangeHolidays(cls, tradeDate): hframe = SignalMgr.getStatic('trading_exchange_holidays_master_frame') hframe = hframe[hframe['tradeDate'] == tradeDate] hframe = hframe[hframe['trading'] == 'No'] lstHolidays = hframe['quoteCountry'].tolist() if 'HK' in hframe['quoteCountry']: lstHolidays = list(set(lstHolidays + ['CN', 'XH'])) return lstHolidays @classmethod def solveProblem(cls, rebalConfig, problem, maxIter=200000, cycleIter=5000, solver=cp.ECOS, verbose=False, checkDCP=False, optimal=False): if checkDCP: dcpFlag = problem.is_dcp() PyLog.info('Problem is DCP (Disciplined Convex Problem): {}'.format(dcpFlag)) PyLog.assertion(dcpFlag, "Optimization problem is not DCP (Disciplined Convex Problem)") if solver == cp.SCS: targetTolerance = rebalConfig.targetTolerance toleranceRange = rebalConfig.toleranceRange nAttempt = 0 maxAttempt = len(toleranceRange) success = False auxIter = 10 numIters = 0 while (not success) and (nAttempt < maxAttempt): maxCycles = int(np.ceil(maxIter / cycleIter)) nCycles = 0 cumIters = 0 while (not success) and (nCycles < maxCycles): tol = targetTolerance try: problem.solve(verbose=verbose, solver=solver, max_iters=cycleIter, eps=tol) except: pass cumIters += problem.solver_stats.num_iters numIters += problem.solver_stats.num_iters nCycles += 1 PyLog.info('tau : {:.2f} / numIters : {} / cumIters : {} / tol : {:e}'.format( 0.0 if optimal else rtau, problem.solver_stats.num_iters, cumIters, tol)) if (problem.solver_stats.num_iters < cycleIter): success = True if (problem.solver_stats.num_iters < cycleIter): success = True else: tol = toleranceRange[nAttempt] try: problem.solve(verbose=verbose, solver=solver, max_iters=auxIter, eps=tol) except: pass PyLog.info('tau : {:.2f} / numIters : {} / tol : {:e}'.format( 0.0 if optimal else rtau, problem.solver_stats.num_iters, tol)) numIters += problem.solver_stats.num_iters if (problem.solver_stats.num_iters < auxIter): success = True else: nAttempt += 1 else: try: success = True PyLog.info(f"level one ECOS solve ") #problem.solve(verbose=verbose, solver=cp.ECOS, max_iters=maxIter, abstol=1e-6, reltol=1e-4, feastol=2e-6, abstol_inacc=2e-6, reltol_inacc=2e-4, feastol_inacc=2e-6) val1 = problem.solve(verbose=verbose, solver=cp.ECOS, max_iters=maxIter, warm_start=True, abstol=1e-5, reltol=5e-4, feastol=1e-5) #CLARABEL : cvxpy deprecated ECOS in May 2024 #PyLog.info(f"level one CLARABEL solve") #problem.solve(verbose=verbose, solver=cp.CLARABEL) except Exception as ee: success = False PyLog.warning(f"Error: optimization failure first level tolerance. td: caught: {ee}") PyLog.warning(traceback.format_exc()) if (not success) or (problem.status == 'infeasible'): try: success = True PyLog.info(f"level two ECOS solve ") #problem.solve(verbose=verbose, solver=cp.ECOS, max_iters=maxIter, # abstol=1e-6, reltol=5e-4, feastol=2e-6, # abstol_inacc=2e-6, reltol_inacc=1e-3, feastol_inacc=2e-6) # --- Level 2 (fallback, looser than Level 1) --- val2 = problem.solve( solver=cp.ECOS, max_iters=int(maxIter * 2), # give the fallback extra room warm_start=True, abstol=5e-5, reltol=1e-3, feastol=2e-5, verbose=verbose ) except Exception as ee: success = False PyLog.warning(f"Error: ECOS failure second level tolerance. td: caught: {ee}") PyLog.warning(traceback.format_exc()) if (not success) or (problem.status == 'infeasible'): try: PyLog.info(f"level three SCS solve ") #problem.solve(verbose=verbose, solver=cp.ECOS, max_iters=maxIter, # abstol=1e-6, reltol=5e-3, feastol=2e-6, # abstol_inacc=2e-6, reltol_inacc=1e-2, feastol_inacc=2e-6) problem.solve(verbose=verbose, solver=cp.SCS) except Exception as ee: PyLog.warning(f"SCS failure third level tolerance. td: caught: {ee}") PyLog.warning(traceback.format_exc()) numIters = problem.solver_stats.num_iters return numIters @classmethod def runOptimal(cls, rebalConfig=RebalConfig_DEFAULT, signalDate=PyDate.asDate(20191115), NAV=100_000_000, solver=cp.SCS, maxIter=200000, cycleIter=5000, verbose=True, checkDCP=True, tradeRestrictions=None): return cls.run(rebalConfig=rebalConfig, signalDate=signalDate, preOptWeights=cls.EMPTY_PORTFOLIO, NAV=NAV, solver=solver, maxIter=maxIter, cycleIter=cycleIter, verbose=verbose, checkDCP=checkDCP, optimal=True, tradeRestrictions=tradeRestrictions) @classmethod def generateRiskReport(cls, dctOptResult, weightCol='postOptWeights', maxReturnDate=PyDate.today()): rebalConfig = dctOptResult['rebalConfig'] portWeights = FrameUtil.toSeries(dctOptResult['dfWeights'], keyCol='assetKey', valCol=weightCol) return RiskReport.runPortfolio(port = portWeights, signalDate = dctOptResult['signalDate'], alphaName = rebalConfig.alphaName, fcostName = rebalConfig.fcostName, bcostName = rebalConfig.bcostName, rmodelName = rebalConfig.rmodelName, envName = Strategy.getModelName(rebalConfig.stratName), minWeight = rebalConfig.holdingThreshold, maxReturnDate = maxReturnDate) @classmethod def printRiskReport(cls, dctOptResult, plotChart=False): dctReport = cls.generateRiskReport(dctOptResult) return RiskReport.print(dctReport, plotChart) @classmethod def compileShortAvailability(cls, rebalConfig, NAV, tradeDate, dframe): lookback = rebalConfig.bcostLookback endTD = tradeDate startTD = PyDate.minusWeekdays(endTD, lookback - 1) dfWgts = pd.DataFrame({'tradeDate': PyDate.sequenceWeekday(startTD, endTD, decreasing=True), 'timeWgts': Stats.expwts(lookback, lookback/2)}) dfWgts = dfWgts.assign(timeScale = dfWgts['timeWgts'] / dfWgts['timeWgts'].iloc[0]) dfRaw = ShortAvailability.getRange(startTD, endTD, pbList=rebalConfig.pbList) dfRaw = dfRaw[dfRaw['assetKey'].isin(dframe['assetKey'])] if rebalConfig.HAUM: AUM = rebalConfig.hypotheticalAUM else: AUM = NAV ########## THIS BLOCK IS NOT NEEDED SINCE WE DON'T HAVE JPM AS BROKER ############ """ dfRaw = dfRaw.assign( scale=rebalConfig.maxShortUtilization + (1 - rebalConfig.maxShortUtilization) * ((dfRaw['pbCode'] == 'JPM') & (dfRaw['category'] == 'connect')).astype(int)) """ ########## THIS BLOCK IS NOT NEEDED SINCE WE DON'T HAVE JPM AS BROKER ############ # So max shorting depends only only this config variable. dfRaw = dfRaw.assign(scale = rebalConfig.maxShortUtilization) dfRaw = dfRaw.assign(vsHAUM=dfRaw['scale'] * dfRaw['notionalUSD'] / AUM) dfRaw = dfRaw.merge(dfWgts[['tradeDate', 'timeScale']], how='left', on='tradeDate') ## availableRate is float and it works poorly with group() dfRaw = dfRaw.assign(rateClass=['{:.4f}'.format(x) for x in dfRaw['availableRate']]) dfMarginal = dfRaw[dfRaw['tradeDate'] == tradeDate].\ groupby('assetKey').agg({'vsHAUM': sum}).reset_index().\ rename(columns={'vsHAUM': 'availablePct'}) ### THIS IS INCORRECTLY SUMMING UP ALL THE EXPONENTIAL DAYS ### #dfConsol = dfRaw.groupby(['assetKey', 'rateClass']). \ #apply(lambda dfm: (dfm['vsHAUM'] * dfm['timeScale']).sum()). \ #reset_index().rename(columns={0: 'availablePct'}) dfConsol = dfRaw.groupby(['assetKey', 'rateClass']) \ .apply(lambda dfm: (dfm['vsHAUM'] * dfm['timeScale']).sum() / dfm['timeScale'].sum()) \ .reset_index().rename(columns={0: 'availablePct'}) dfConsol = dfConsol.assign(availableRate=dfConsol['rateClass'].astype(float)) dfConsol = dfConsol.sort_values(by=['assetKey', 'availableRate'], ascending=True). \ groupby('assetKey').apply(lambda dfm: dfm.assign(cumHi=dfm['availablePct'].cumsum())).reset_index(drop=True) dfConsol = dfConsol.groupby('assetKey'). \ apply(lambda dfm: dfm.assign(cumLo=[0.0] + dfm['cumHi'][:-1].tolist())).reset_index(drop=True) dfConsol = dfConsol.merge(dframe[['assetKey', 'preOptWeights']], how='left', on='assetKey') ## discard irrelevant availability softShortBoundBuffer = 0.01 dfConsol = dfConsol.merge(dframe[['assetKey', 'shortBoundSoft', 'shortBoundHard']], how='left', on='assetKey') dfConsol = dfConsol.assign(bufferedShortBoundSoft=dfConsol['shortBoundSoft'] + softShortBoundBuffer) dfConsol = dfConsol[dfConsol['cumLo'] <= dfConsol['shortBoundHard']] dfConsol = dfConsol[dfConsol['cumLo'] <= dfConsol['bufferedShortBoundSoft']] ## technically, this is not strict but we are going to simplify the logic and keep track of ## all availability and marginal availability dfConsol = dfConsol.sort_values(by=['assetKey', 'availableRate'], ascending=True). \ groupby('assetKey').apply(lambda dfm: dfm.assign(tier=range(len(dfm)))).reset_index(drop=True) columns = ['assetKey', 'tier', 'availableRate', 'availablePct'] dfBaseTier = dfConsol[dfConsol['tier'] == 0][columns].merge(dframe, how='inner', on='assetKey') #dfBaseTier = dfBaseTier.assign(bcost=dfBaseTier['availableRate'] / 100) dfBaseTier = dfBaseTier.assign(bcost=dfBaseTier['availableRate'] ) # dfBaseTier = dfBaseTier.assign(shortBoundHard = dfBaseTier[['shortBoundHard', 'availablePct']].min(axis=1)) dfBaseTier = dfBaseTier.assign(shortBoundHard=dfBaseTier['availablePct']) dfBaseTier = dfBaseTier.drop(columns=['availableRate', 'availablePct']) dfHighTier = dfConsol[dfConsol['tier'] != 0][['assetKey', 'tier', 'availableRate', 'availablePct']]. \ merge(dframe, how='inner', on='assetKey') dfHighTier = dfHighTier.assign(longBoundHard=0.0) dfHighTier = dfHighTier.assign(shortBoundHard=dfHighTier['availablePct']) #dfHighTier = dfHighTier.assign(bcost=dfHighTier['availableRate'] / 100) dfHighTier = dfHighTier.assign(bcost=dfHighTier['availableRate'] ) dfHighTier = dfHighTier.assign(fcost=1.00) dfHighTier = dfHighTier.assign(preOptWeights=np.NaN) dfHighTier = dfHighTier.drop(columns=['availableRate', 'availablePct']) ## no short availability dfNoAvail = dframe[~dframe['assetKey'].isin(dfConsol['assetKey'])] dfNoAvail = dfNoAvail.assign(shortBoundHard=0.0) dfNoAvail = dfNoAvail.assign(bcost=1.0) dfNoAvail = dfNoAvail.assign(tier=-1) dfAvail = pd.concat([dfBaseTier, dfHighTier, dfNoAvail], ignore_index=True) dfAvail = dfAvail.assign(key=dfAvail['assetKey'] + ':' + dfAvail['tier'].astype(str)) dfAvail = dfAvail.sort_values(by=['assetKey', 'tier']) ## bcostMultiplier if rebalConfig.bcostMultiplierName is not None: signalDate = PyDate.prevWeekday(tradeDate) signalObj = Signal.registryLookup(rebalConfig.bcostMultiplierName) dfm = SignalMgr.getFrame(signalObj.signalName, signalDate).\ rename(columns={signalObj.signalName: 'multiplier'}) dfAvail = dfAvail.merge(dfm[['assetKey', 'multiplier']], how='left', on='assetKey') dfAvail = dfAvail.assign(multiplier = dfAvail['multiplier'].fillna(signalObj.maxMultiplier)) dfAvail = dfAvail.assign(bcost = dfAvail['bcost'] * dfAvail['multiplier']) dfAvail = dfAvail.drop(columns='multiplier') return dfAvail, dfMarginal def load_china_connect_inventory_scores(input_date, max_lookback_days): """ Try to read China Connect Inventory Score file for the given signal date. From what I have noticed, for tradeDate say 11/19, we get files for 11/18 and that too at 9pm NY time but our optimizer runs at 3pm NY time so these 11/18 files come late. So we process only two days earlier files for 11/17. Hence I have put in logic for max_lookback_days as 3 so we will try to get trade_date(say 11/19) files, if not (11/18), if not (11/17) Returns dataframe ['assetKey','stabilityScore'] or None. """ import PyUtil.PyRicX as RicX try: pbdata_dir = os.environ.get('PBDIR') if not pbdata_dir: PyLog.info("ChinaConnect: PBDIR env var not set; skipping China Connect short penalty.") return None base_dir = os.path.join(pbdata_dir, "GS_Stability_Files") if not os.path.isdir(base_dir): PyLog.info(f"ChinaConnect: directory {base_dir} does not exist; skipping China Connect short penalty.") return None # ---------------------------------------------------------- # NEW: Loop over working days (signalDate, -1 WD, -2 WD, etc.) # ---------------------------------------------------------- dates_to_try = [] d = PyDate.asDate(input_date) for _ in range(max_lookback_days): dates_to_try.append(d) d = PyDate.prevWeekday(d) filename = None for d in dates_to_try: date_str = d.strftime("%Y%m%d") pattern_xls = os.path.join(base_dir, f"*China_Connect*In*{date_str}.xls") pattern_xlsx = os.path.join(base_dir, f"*China_Connect*In*{date_str}.xlsx") files = glob.glob(pattern_xls) + glob.glob(pattern_xlsx) if files: files.sort(key=os.path.getmtime) filename = files[-1] # newest file for that date PyLog.info(f"ChinaConnect: using inventory score file {filename} (date {date_str})") break else: PyLog.info(f"ChinaConnect: no file for date {date_str}, trying previous working day...") if not filename: PyLog.info(f"ChinaConnect: no China_Connect file found in last {max_lookback_days} working days; skipping penalty.") return None # ---------------------------------------------------------- # Read without header; find the row where the table starts raw = pd.read_excel(filename, header=None) col0 = raw.iloc[:, 0].astype(str).str.strip() header_rows = raw.index[col0 == "RIC"].tolist() if not header_rows: PyLog.info(f"ChinaConnect: could not find header row with 'RIC' in {filename}; skipping penalty.") return None h = header_rows[0] df = raw.iloc[h:].copy() df.columns = df.iloc[0] df = df.iloc[1:] # drop header row df = df.rename(columns={c: str(c).strip() for c in df.columns}) expected_cols = {"RIC", "SCORE", "SEDOL", "ISIN"} if not expected_cols.issubset(df.columns): PyLog.info(f"ChinaConnect: expected columns {expected_cols} not found; skipping penalty.") return None df = df[["RIC", "SCORE", "SEDOL", "ISIN"]].copy() df["RIC"] = df["RIC"].astype(str).str.strip() df["SCORE"] = pd.to_numeric(df["SCORE"], errors="coerce") # Keep only ZK and SH mask = df["RIC"].str.endswith(".ZK") | df["RIC"].str.endswith(".SH") df = df[mask].copy() if df.empty: PyLog.info("ChinaConnect: no .ZK/.SH rows in file; nothing to do.") return None # Convert to internal_ric: ZK->SZ, SH->SS df["internal_ric"] = df["RIC"] df.loc[df["RIC"].str.endswith(".ZK"), "internal_ric"] = df["RIC"].str.replace(".ZK", ".SZ", regex=False) df.loc[df["RIC"].str.endswith(".SH"), "internal_ric"] = df["RIC"].str.replace(".SH", ".SS", regex=False) # Map to assetKey df = RicX.add_asset_code(df, "internal_ric") if "assetKey" not in df.columns: PyLog.info("ChinaConnect: add_asset_code failed; skipping penalty.") return None df = df[["assetKey", "SCORE"]].dropna() if df.empty: PyLog.info("ChinaConnect: no usable rows remain; skipping penalty.") return None # Aggregate duplicates by max score df = df.groupby("assetKey", as_index=False)["SCORE"].max() df = df.rename(columns={"SCORE": "stabilityScore"}) PyLog.info(f"ChinaConnect: loaded {len(df)} inventory score rows with assetKey.") return df except Exception as e: PyLog.error(f"ChinaConnect: exception while loading inventory scores: {e}") PyLog.error(traceback.format_exc()) return None def build_china_connect_short_penalty(dfAvail, wshort, input_date): """ Build a cvxpy penalty term for short positions based on China Connect stability scores. - Only applies to assets in dfAvail (optimizer universe in split block). - Only applies on the short side (wshort). - No penalty for score <= 4. - For scores 5–8, penalty increases linearly with (score - 4). - Logs which assets are penalized (pre-solve) and stores metadata for post-solve debug logging. Returns: cp.Expression or None """ # For now we will hardcode max_lookback_days here, ideally it should be in rebalance.config max_lookback_days = 3 # Load scores from file df_scores = load_china_connect_inventory_scores(input_date, max_lookback_days) if df_scores is None or df_scores.empty: return None # Map assetKey -> stabilityScore df_scores = df_scores.set_index("assetKey")["stabilityScore"] if "assetKey" not in dfAvail.columns: PyLog.info("ChinaConnect: dfAvail has no 'assetKey' column; cannot apply stability penalty.") return None n = len(dfAvail) penalties = np.zeros(n) penalized_assets = [] # (idx, assetKey, score, penaltyWeight) # Build penalty weights per dfAvail row for idx, a in enumerate(dfAvail["assetKey"]): if a in df_scores.index: s = df_scores.loc[a] if pd.notna(s) and s > 4: # 5→1, 6→2, 7→3, 8→4 (can be tuned) pw = max(0.0, float(s) - 4.0) penalties[idx] = pw penalized_assets.append((idx, a, float(s), pw)) if not penalized_assets: PyLog.info("ChinaConnect: no dfAvail assets with stabilityScore > 4; no penalty applied.") return None # Log penalized assets BEFORE solve (no short weights yet) PyLog.info(f"ChinaConnect: {len(penalized_assets)} dfAvail assets will have short-side stability penalty.") PyLog.info("ChinaConnect: penalized assets (before solve, top 50):") for idx, a, s, pw in penalized_assets[:50]: PyLog.info(f" idx={idx}, assetKey={a}, score={s}, penaltyWeight={pw}") if len(penalized_assets) > 50: PyLog.info(f" ...and {len(penalized_assets) - 50} more penalized assets") # Optional tuning knob (strength of this penalty) china_scale = 0.1 # adjust as needed or make configurable # penalties is constant data; cvxpy treats it as coefficients penalties_vec = penalties.reshape(-1, 1) penalty_expr = china_scale * cp.sum(cp.multiply(penalties_vec, wshort)) # Attach metadata so we can inspect after solve penalty_expr._china_penalty_info = penalized_assets penalty_expr._china_penalty_weights = penalties_vec penalty_expr._china_scale = china_scale PyLog.info("ChinaConnect: short-side stability penalty term added to objective.") return penalty_expr def build_trbc_cluster_penalty( dframe, assets, aggrw, tradeDate, gqr_dir="/data/gqr_specret_cov", corr_threshold=0.5, min_cluster_size=2, lambda_cluster=0.5, ): """ Build a soft penalty that discourages large net exposure in highly correlated TRBC-activity clusters, while allowing hedging within the cluster. Uses TRBC activity from dframe['Activity'] and GQR var/scov files. Parameters ---------- dframe : DataFrame Current universe with at least ['assetKey', 'Activity'] and 'idx' mapping. assets : array-like AssetKey list in the same order as aggrw (the optimization universe). aggrw : cp.Variable N x 1 cvxpy variable of aggregate weights. tradeDate : date Current trade date. gqr_dir : str Directory where gqr_specific_var.*.csv and gqr_scov_sparse.*.csv live. corr_threshold : float Minimum |corr| inside activity to treat a pair as 'highly correlated'. min_cluster_size : int Ignore clusters smaller than this (e.g. singletons). lambda_cluster : float Strength of the penalty. Larger = stronger push towards neutrality. Returns ------- cp.Expression (scalar) or 0 The penalty term to be added to the global 'penalty'. """ activity_col = "Activity" # If we don't have activity information, bail out quietly if activity_col not in dframe.columns: PyLog.info(f"TRBC cluster penalty: column {activity_col} not found → skipping") return 0 date_str = tradeDate.strftime("%Y%m%d") var_path = os.path.join(gqr_dir, f"gqr_specific_var.{date_str}.csv") scov_path = os.path.join(gqr_dir, f"gqr_scov_sparse.{date_str}.csv") if not (os.path.exists(var_path) and os.path.exists(scov_path)): PyLog.info(f"TRBC cluster penalty: missing var/scov for {date_str} → skipping") return 0 try: var_df = pd.read_csv(var_path) var_df = var_df.set_index("assetKey")["spec_var"] scov_df = pd.read_csv(scov_path) except Exception as e: PyLog.info(f"TRBC cluster penalty: failed to read var/scov for {date_str}: {e} → skipping") return 0 # Map assetKey -> activity for current universe df_act = ( dframe[["assetKey", activity_col]] .dropna() .drop_duplicates(subset=["assetKey"]) .set_index("assetKey") ) act_map = df_act[activity_col].to_dict() # Map assetKey -> index in aggrw key_to_idx = {k: i for i, k in enumerate(assets)} # Build adjacency list per activity from high-corr pairs # adjacency[activity][idx] = set(neighbor_idx, ...) adjacency = {} # Restrict scov to pairs that are in our asset universe and have activity info scov_df = scov_df[ scov_df["xKey"].isin(key_to_idx.keys()) & scov_df["yKey"].isin(key_to_idx.keys()) ] # Precompute correlations and filter on |corr| >= corr_threshold edges_used = 0 for _, row in scov_df.iterrows(): x = row["xKey"] y = row["yKey"] ax = act_map.get(x, None) ay = act_map.get(y, None) if ax is None or ay is None: continue # Only within same TRBC activity if ax != ay: continue vx = var_df.get(x, None) vy = var_df.get(y, None) if vx is None or vy is None or vx <= 0 or vy <= 0: continue cov_xy = row["cov"] corr_xy = cov_xy / math.sqrt(vx * vy) if abs(corr_xy) < corr_threshold: continue i = key_to_idx[x] j = key_to_idx[y] if ax not in adjacency: adjacency[ax] = {} if i not in adjacency[ax]: adjacency[ax][i] = set() if j not in adjacency[ax]: adjacency[ax][j] = set() adjacency[ax][i].add(j) adjacency[ax][j].add(i) edges_used += 1 if edges_used == 0: PyLog.info(f"TRBC cluster penalty: no within-activity high-corr pairs for {date_str} → skipping") return 0 # Find connected components (clusters) within each activity clusters = [] for act, adj in adjacency.items(): visited = set() for node in list(adj.keys()): if node in visited: continue # BFS / DFS to collect a cluster stack = [node] comp = [] visited.add(node) while stack: u = stack.pop() comp.append(u) for v in adj[u]: if v not in visited: visited.add(v) stack.append(v) if len(comp) >= min_cluster_size: clusters.append((act, comp)) if len(clusters) == 0: PyLog.info(f"TRBC cluster penalty: no clusters >= {min_cluster_size} for {date_str} → skipping") return 0 # --- DEBUG: dump clusters to CSV with activity, assetKey, RIC, etc. --- try: cluster_rows = [] # Build a quick lookup for extra fields (RIC, Name, etc.) extra_cols = [] for cand in ["RkdDisplayRIC", "RIC", "Name"]: if cand in dframe.columns: extra_cols.append(cand) # We'll index dframe by assetKey once for speed df_by_key = dframe.set_index("assetKey") for cid, (act, idx_list) in enumerate(clusters): for idx in idx_list: asset = assets[idx] row = { "date": date_str, "activity": act, "cluster_id": cid, "assetKey": asset, } if asset in df_by_key.index: rec = df_by_key.loc[asset] for col in extra_cols: row[col] = rec.get(col, None) cluster_rows.append(row) if cluster_rows: df_clusters = pd.DataFrame(cluster_rows) out_path = os.path.join(gqr_dir, f"gqr_trbc_clusters.{date_str}.csv") df_clusters.to_csv(out_path, index=False) PyLog.info( f"TRBC cluster penalty: wrote {len(df_clusters)} rows for {len(clusters)} clusters " f"to {out_path}" ) except Exception as e: PyLog.info(f"TRBC cluster penalty: failed to write clusters CSV for {date_str}: {e}") # Build the cvxpy penalty: lambda * sum_C (sum_{i in C} w_i)^2 penalty_terms = [] for act, idx_list in clusters: exposure_C = cp.sum(aggrw[idx_list]) penalty_terms.append(cp.square(exposure_C)) if not penalty_terms: return 0 pen_expr = lambda_cluster * cp.sum(penalty_terms) PyLog.info( f"TRBC cluster penalty: built {len(clusters)} clusters from {edges_used} edges for {date_str}, " f"lambda={lambda_cluster}" ) return pen_expr def log_trbc_cluster_exposures( tradeDate, assets, aggrw, gqr_dir="/data/gqr_specret_cov", lambda_cluster=None, ): """ After the solve, log per-cluster exposures and (optionally) the total TRBC cluster penalty implied by the current weights. We log: - raw weights (fractions of NAV), - percentages of NAV (much more readable), - and, if lambda_cluster is given, the implied penalty. aggrw: cvxpy expression of aggregate weights per stock (same scale as portfolio weights). """ try: date_str = tradeDate.strftime("%Y%m%d") cluster_path = os.path.join(gqr_dir, f"gqr_trbc_clusters.{date_str}.csv") if not os.path.exists(cluster_path): PyLog.info(f"TRBC cluster exposure: no cluster file for {date_str} at {cluster_path} → skipping") return w = aggrw.value if w is None: PyLog.info("TRBC cluster exposure: aggrw.value is None ... skipping") return w = np.asarray(w).reshape(-1) df_cl = pd.read_csv(cluster_path) # Map assetKey -> index in aggrw key_to_idx = {k: i for i, k in enumerate(assets)} weights = [] for _, row in df_cl.iterrows(): ak = row["assetKey"] idx = key_to_idx.get(ak) if idx is None: weights.append(np.nan) else: weights.append(w[idx]) df_cl["weight"] = weights # raw weight (fraction of NAV) df_cl["abs_weight"] = df_cl["weight"].abs() # Also in percent of NAV (much easier to read) df_cl["weight_pct"] = df_cl["weight"] * 100.0 df_cl["abs_weight_pct"] = df_cl["abs_weight"] * 100.0 # Aggregate to cluster level grp = df_cl.groupby(["date", "activity", "cluster_id"], as_index=False).agg( n_assets=("assetKey", "nunique"), # raw exposures (fractions of NAV) cluster_net_w=("weight", "sum"), cluster_gross_w=("abs_weight", "sum"), max_gross_w=("abs_weight", "max"), # percent of NAV exposures (more interpretable) cluster_net_w_pct=("weight_pct", "sum"), cluster_gross_w_pct=("abs_weight_pct", "sum"), max_gross_w_pct=("abs_weight_pct", "max"), ) total_penalty = None max_cluster_pen = None # If lambda is provided, compute penalty per cluster and total if lambda_cluster is not None: grp["cluster_penalty"] = lambda_cluster * (grp["cluster_net_w"] ** 2) total_penalty = float(grp["cluster_penalty"].sum()) max_cluster_pen = float(grp["cluster_penalty"].max()) # For quick eyeballing, also get the worst cluster's net exposure (pct) max_net_pct = float(grp["cluster_net_w_pct"].abs().max()) max_gross_pct = float(grp["cluster_gross_w_pct"].max()) PyLog.info( f"TRBC cluster exposure: date={date_str}, " f"lambda={lambda_cluster}, " f"total_cluster_penalty={total_penalty:.6f}, " f"max_cluster_penalty={max_cluster_pen:.6f}, " f"max_cluster_net_w_pct={max_net_pct:.2f}%, " f"max_cluster_gross_w_pct={max_gross_pct:.2f}%" ) # Append a one-line summary CSV (easier for later analysis) summary_path = os.path.join(gqr_dir, "gqr_trbc_cluster_penalty_summary.csv") header_needed = not os.path.exists(summary_path) with open(summary_path, "a") as f: if header_needed: f.write( "date,lambda,total_penalty,max_cluster_penalty," "max_cluster_net_w_pct,max_cluster_gross_w_pct\n" ) f.write(f"{date_str},{lambda_cluster}," f"{total_penalty:.6f},{max_cluster_pen:.6f}," f"{max_net_pct:.2f},{max_gross_pct:.2f}\n" ) expo_path = os.path.join(gqr_dir, f"gqr_trbc_cluster_exposure.{date_str}.csv") round_cols = [ "cluster_net_w_pct", "cluster_gross_w_pct", "max_gross_w_pct", "cluster_penalty", ] for c in round_cols: if c in grp.columns: grp[c] = grp[c].round(2) grp.to_csv(expo_path, index=False) if total_penalty is None: PyLog.info( f"TRBC cluster exposure: wrote {len(grp)} rows to {expo_path}" ) else: PyLog.info( f"TRBC cluster exposure: wrote {len(grp)} rows to {expo_path}, " f"total_penalty={total_penalty:.6g}" ) except Exception as e: PyLog.info(f"TRBC cluster exposure: failed for {tradeDate}: {e}")