import cvxpy as cp import copy from numpy import linalg as LA import os import re import traceback import glob import numpy as np import pandas as pd import math from openpyxl import load_workbook from ..signal.engine import * from ..signal.interface import * from ..risk import * from ..domain import * from model.trading.EG import EG from model.trading.process_gqr_specret import process_gqr_specret """ from model.trading.periscope_alpha import ( applyPeriscopeOverlay, applyPeriscopeAlphaConditioning, ) """ from model.trading.build_simple_alpha import get_simple_alpha # === Axioma specific covariance integration === GQR_SPECRET_DIR = "/data/gqr_specret_cov" # Set pandas display options pd.set_option('display.max_rows', None) # Show all rows pd.set_option('display.max_columns', None) # Show all columns pd.set_option('display.width', 1000) # Set wider console width pd.set_option('display.max_colwidth', None) # Show full content in each column pd.set_option('display.float_format', '{:.4f}'.format) # Show floating point numbers with 4 decimals def matrix_shrink(Q): """ Take Q and reapprox using subset of singular values """ # shrink matrix U, S, V = LA.svd(Q) num_sing_values = S.shape[0] U = U[:, :num_sing_values] # log some singular values PyLog.info(f"matrix_shrink sing values top5 {S[0:5]} bot5 {S[-5:]}") # use at least portion of the singular values, and always at least K but not more than the rank of the matrix # very cut down settings we have used in some tests: portion= 1/8; K = 10 # typical keep most settings portion= 7/8; K= 20 portion = 7/8 K = 20 num_sing_vaule_to_use = max(int(num_sing_values*portion), K) r = min(num_sing_values, num_sing_vaule_to_use) PyLog.info(f"matrix_shrink got {num_sing_values} singular values. portion:{portion:.3f} K:{K} reconstructing with {r}. matrix shpe {Q.shape} first/last {S[0]/S[4] :.2f}") S = np.diag(S) R = U[:, :r] @ S[:r, :r] @ V[:r, :] return R class RebalConfig: dctDefaultOptParams = { 'SCS': {'maxIter': 200_000, 'cycleIter': 5_000, 'targetTolerance': 1e-4, 'toleranceRange': [7e-4, 1e-3, 2e-3, 3e-3, 4e-3, 5e-3]}, 'ECOS': {'maxIter': 20_000} } DEFAULT_CONFIG = {'configName': 'DEFAULT', 'stratName': 'strat_asia_v1', 'maxLeverage': +4.00, 'minNetExposure': -0.05, 'maxNetExposure': +0.40, 'longOnlyMinNetExposure': +0.99, 'longOnlyMaxNetExposure': +1.00, 'longOnlyMinBetaAdjNet': +0.75, 'longOnlyMaxBetaAdjNet': +1.25, 'minBetaAdjNet': -0.01, 'maxBetaAdjNet': +0.01, 'gamma': 1.8, 'adaptiveGamma': False, 'adaptiveGammaTargetRisk': 0.10, 'adaptiveGammaTolerance': 0.0025, 'tau': 4.0, 'maxRisk': 0.13, 'offDiagRisk': False, 'splitLongShort': False, 'splitLongShortStartDate': PyDate.asDate(20220615), 'bcostLookback': 8, 'maxShortUtilization': 0.8, 'pbList': None, 'bcostMultiplierName': None, 'HAUM': False, 'hypotheticalAUM': 100_000_000, 'univwtThreshold': 0.25, 'holdingThreshold': 0.00005, 'rmodelName': 'risk_model', 'tmodelName': 'tcost_model', 'impactCostScale': 1.0, 'alphaName': 'alpha', 'tradeUniverseName': None, 'themeBoundName': None, 'fcostName': 'trading_long_financing_cost', 'bcostName': 'trading_short_borrow_cost', 'defaultFCost': 0.005, 'defaultBCost': 0.15, 'longBoundName': 'trading_position_bounds_long', 'shortBoundName': 'trading_position_bounds_short', 'defaultLongBound': 0.000025, 'defaultShortBound': 0.000025, 'liquidityName': 'adv_usdm_blended', 'defaultLiquidityUSDM': 1.0, 'maxLiquidityBoundLong': None, 'maxLiquidityBoundShort': None, 'applyShortBan': True, 'applyExchangeHolidays': False, 'advName': 'adv_usdm_blended', 'countryBoundName': 'trading_country_bounds_frame', 'industryBoundName': 'trading_industry_bounds_frame', 'sizeBoundName': 'trading_size_bounds_frame', 'targetTolerance': 1e-4, 'toleranceRange': [7e-4, 1e-3, 2e-3, 3e-3, 4e-3, 5e-3], 'minShortAvailM': 0, 'shrinkfcov' : False, 'enable_gs_stability_score' : False, 'initializeEmptyRebalDateAsStartTD':False, 'enable_3ai_alpha':False, 'three_ai_data_dir':None, 'top_bottom_pct_3ai_stocks':100, 'periscopeExcludedCountries' : None, 'optimizer_countries':None, 'enable_alpha_conditioning':True, 'enable_stock_selection_alpha':False, 'stock_selection_alpha_countries' : None, 'enable_simple_alpha':False, 'strategyLongShortType':"BOTH", 'simple_alpha_names' : None, 'simple_alpha_countries' : None, 'three_ai_alpha_countries' : None, 'subdir':None, 'periscopeMode': 'UNWIND', # OFF | UNWIND | HARD | SOFT_PENALTY 'periscopeDir': '/data/periscope', 'periscopeWindowDays': 30, 'periscopeFadeFallbackDays': 10, 'periscopeMinAbsDaysToTrade': 1.0, 'periscopeMinConviction': 'High', 'periscopeEnableLogging': True, 'periscopeLogDir': '/data/log/periscope', 'periscopeLogIncludeChangeSheet': False, 'periscopePenaltyLambda': 5.0, 'tightCountry' : False,} @classmethod def defaultConfig(cls, parameter): return cls.DEFAULT_CONFIG[parameter] def __init__(self, configName = None, stratName = None, maxLeverage = None, minNetExposure = None, maxNetExposure = None, longOnlyMinNetExposure=None, longOnlyMaxNetExposure=None, longOnlyMinBetaAdjNet=None, longOnlyMaxBetaAdjNet=None, minBetaAdjNet = None, maxBetaAdjNet = None, gamma = None, adaptiveGamma = None, adaptiveGammaTargetRisk = None, adaptiveGammaTolerance = None, tau = None, maxRisk = None, offDiagRisk = None, splitLongShort = None, splitLongShortStartDate = None, maxShortUtilization = None, pbList = None, bcostMultiplierName = None, HAUM = None, hypotheticalAUM = None, bcostLookback = None, univwtThreshold = None, holdingThreshold = None, rmodelName = None, tmodelName = None, impactCostScale = None, alphaName = None, tradeUniverseName = None, themeBoundName = None, fcostName = None, bcostName = None, defaultFCost = None, defaultBCost = None, longBoundName = None, shortBoundName = None, liquidityName = None, defaultLiquidityUSDM = None, maxLiquidityBoundLong = None, maxLiquidityBoundShort = None, defaultLongBound = None, defaultShortBound = None, applyShortBan = None, applyExchangeHolidays = None, advName = None, maxAdvProp = None, countryBoundName = None, industryBoundName = None, sizeBoundName = None, targetTolerance = None, toleranceRange = None, minShortAvailM = None, shrinkfcov = None, enable_gs_stability_score = None, initializeEmptyRebalDateAsStartTD=None, enable_3ai_alpha=None, three_ai_data_dir=None, top_bottom_pct_3ai_stocks=None, periscopeExcludedCountries = None, optimizer_countries = None, enable_alpha_conditioning=None, enable_stock_selection_alpha=None, stock_selection_alpha_countries = None, enable_simple_alpha=None, strategyLongShortType=None, simple_alpha_names = None, simple_alpha_countries = None, three_ai_alpha_countries = None, subdir=None, tightCountry = None,): self.configName = self.defaultConfig('configName') if configName is None else configName self.stratName = self.defaultConfig('stratName') if stratName is None else stratName self.maxLeverage = self.defaultConfig('maxLeverage') if maxLeverage is None else maxLeverage self.minNetExposure = self.defaultConfig('minNetExposure') if minNetExposure is None else minNetExposure self.maxNetExposure = self.defaultConfig('maxNetExposure') if maxNetExposure is None else maxNetExposure self.longOnlyMinNetExposure = (self.defaultConfig('longOnlyMinNetExposure') if longOnlyMinNetExposure is None else longOnlyMinNetExposure) self.longOnlyMaxNetExposure = (self.defaultConfig('longOnlyMaxNetExposure') if longOnlyMaxNetExposure is None else longOnlyMaxNetExposure) self.longOnlyMaxBetaAdjNet = (self.defaultConfig('longOnlyMaxBetaAdjNet') if longOnlyMaxBetaAdjNet is None else longOnlyMaxBetaAdjNet) self.longOnlyMinBetaAdjNet = (self.defaultConfig('longOnlyMinBetaAdjNet') if longOnlyMinBetaAdjNet is None else longOnlyMinBetaAdjNet) self.minBetaAdjNet = self.defaultConfig('minBetaAdjNet') if minBetaAdjNet is None else minBetaAdjNet self.maxBetaAdjNet = self.defaultConfig('maxBetaAdjNet') if maxBetaAdjNet is None else maxBetaAdjNet self.gamma = self.defaultConfig('gamma') if gamma is None else gamma self.adaptiveGamma = self.defaultConfig('adaptiveGamma') if adaptiveGamma is None else adaptiveGamma self.adaptiveGammaTargetRisk = self.defaultConfig('adaptiveGammaTargetRisk')\ if adaptiveGammaTargetRisk is None else adaptiveGammaTargetRisk self.adaptiveGammaTolerance = self.defaultConfig('adaptiveGammaTolerance')\ if adaptiveGammaTolerance is None else adaptiveGammaTolerance self.tau = self.defaultConfig('tau') if tau is None else tau self.maxRisk = self.defaultConfig('maxRisk') if maxRisk is None else maxRisk self.offDiagRisk = self.defaultConfig('offDiagRisk') if offDiagRisk is None else offDiagRisk self.splitLongShort = self.defaultConfig('splitLongShort') if splitLongShort is None else splitLongShort self.splitLongShortStartDate = self.defaultConfig('splitLongShortStartDate')\ if splitLongShortStartDate is None else splitLongShortStartDate self.maxShortUtilization = self.defaultConfig('maxShortUtilization')\ if maxShortUtilization is None else maxShortUtilization self.pbList = self.defaultConfig('pbList') if pbList is None else pbList self.bcostMultiplierName = self.defaultConfig('bcostMultiplierName')\ if bcostMultiplierName is None else bcostMultiplierName self.HAUM = self.defaultConfig('HAUM') if HAUM is None else HAUM self.hypotheticalAUM = self.defaultConfig('hypotheticalAUM')\ if hypotheticalAUM is None else hypotheticalAUM self.bcostLookback = self.defaultConfig('bcostLookback') if bcostLookback is None else bcostLookback self.univwtThreshold = self.defaultConfig('univwtThreshold') if univwtThreshold is None else univwtThreshold self.holdingThreshold = self.defaultConfig('holdingThreshold') if holdingThreshold is None else holdingThreshold self.rmodelName = self.defaultConfig('rmodelName') if rmodelName is None else rmodelName self.tmodelName = self.defaultConfig('tmodelName') if tmodelName is None else tmodelName self.impactCostScale = self.defaultConfig('impactCostScale') if impactCostScale is None else impactCostScale self.alphaName = self.defaultConfig('alphaName') if alphaName is None else alphaName self.tradeUniverseName = \ self.defaultConfig('tradeUniverseName') if tradeUniverseName is None else tradeUniverseName #self.themeBoundName = themeBoundName self.themeBoundName = self.defaultConfig('themeBoundName') if themeBoundName is None else themeBoundName self.fcostName = self.defaultConfig('fcostName') if fcostName is None else fcostName self.bcostName = self.defaultConfig('bcostName') if bcostName is None else bcostName self.defaultFCost = self.defaultConfig('defaultFCost') if defaultFCost is None else defaultFCost self.defaultBCost = self.defaultConfig('defaultBCost') if defaultBCost is None else defaultBCost self.longBoundName = self.defaultConfig('longBoundName') if longBoundName is None else longBoundName self.shortBoundName = self.defaultConfig('shortBoundName') if shortBoundName is None else shortBoundName self.defaultLongBound = \ self.defaultConfig('defaultLongBound') if defaultLongBound is None else defaultLongBound self.defaultShortBound = \ self.defaultConfig('defaultShortBound') if defaultShortBound is None else defaultShortBound self.liquidityName = \ self.defaultConfig('liquidityName') if liquidityName is None else liquidityName self.defaultLiquidityUSDM = \ self.defaultConfig('defaultLiquidityUSDM') if defaultLiquidityUSDM is None else defaultLiquidityUSDM self.maxLiquidityBoundLong = \ self.defaultConfig('maxLiquidityBoundLong') if maxLiquidityBoundLong is None else maxLiquidityBoundLong self.maxLiquidityBoundShort = \ self.defaultConfig('maxLiquidityBoundShort') if maxLiquidityBoundShort is None else maxLiquidityBoundShort self.applyShortBan = self.defaultConfig('applyShortBan') if applyShortBan is None else applyShortBan self.applyExchangeHolidays = \ self.defaultConfig('applyExchangeHolidays') if applyExchangeHolidays is None else applyExchangeHolidays self.advName = self.defaultConfig('advName') if advName is None else advName self.maxAdvProp = maxAdvProp self.countryBoundName = \ self.defaultConfig('countryBoundName') if countryBoundName is None else countryBoundName self.industryBoundName = \ self.defaultConfig('industryBoundName') if industryBoundName is None else industryBoundName self.sizeBoundName = self.defaultConfig('sizeBoundName') if sizeBoundName is None else sizeBoundName self.targetTolerance = self.defaultConfig('targetTolerance') if targetTolerance is None else targetTolerance self.toleranceRange = self.defaultConfig('toleranceRange') if toleranceRange is None else toleranceRange self.minShortAvailM = self.defaultConfig('minShortAvailM') if minShortAvailM is None else minShortAvailM self.shrinkfcov = self.defaultConfig('shrinkfcov') if shrinkfcov is None else shrinkfcov self.tightCountry = self.defaultConfig('tightCountry') if tightCountry is None else tightCountry self.enable_gs_stability_score = self.defaultConfig('enable_gs_stability_score') if enable_gs_stability_score is None else enable_gs_stability_score self.initializeEmptyRebalDateAsStartTD = self.defaultConfig('initializeEmptyRebalDateAsStartTD') if initializeEmptyRebalDateAsStartTD is None else initializeEmptyRebalDateAsStartTD self.enable_3ai_alpha = self.defaultConfig('enable_3ai_alpha') if enable_3ai_alpha is None else enable_3ai_alpha self.three_ai_data_dir = self.defaultConfig('three_ai_data_dir') if three_ai_data_dir is None else three_ai_data_dir self.strategyLongShortType = self.defaultConfig('strategyLongShortType') if strategyLongShortType is None else strategyLongShortType self.optimizer_countries = self.defaultConfig('optimizer_countries') if optimizer_countries is None else optimizer_countries self.enable_alpha_conditioning = self.defaultConfig('enable_alpha_conditioning') if enable_alpha_conditioning is None else enable_alpha_conditioning self.enable_stock_selection_alpha = self.defaultConfig('enable_stock_selection_alpha') if enable_stock_selection_alpha is None else enable_stock_selection_alpha self.stock_selection_alpha_countries = self.defaultConfig('stock_selection_alpha_countries') if stock_selection_alpha_countries is None else stock_selection_alpha_countries self.enable_simple_alpha = self.defaultConfig('enable_simple_alpha') if enable_simple_alpha is None else enable_simple_alpha self.simple_alpha_names = self.defaultConfig('simple_alpha_names') if simple_alpha_names is None else simple_alpha_names self.simple_alpha_countries = self.defaultConfig('simple_alpha_countries') if simple_alpha_countries is None else simple_alpha_countries self.three_ai_alpha_countries = self.defaultConfig('three_ai_alpha_countries') if three_ai_alpha_countries is None else three_ai_alpha_countries self.top_bottom_pct_3ai_stocks = self.defaultConfig('top_bottom_pct_3ai_stocks') if top_bottom_pct_3ai_stocks is None else top_bottom_pct_3ai_stocks self.subdir = subdir self.periscopeExcludedCountries = self.defaultConfig('periscopeExcludedCountries') if periscopeExcludedCountries is None else periscopeExcludedCountries RebalConfig_DEFAULT = RebalConfig() ## ================================================================================================================== class Rebalance: EMPTY_PORTFOLIO = pd.Series(1.0, index=['$USD']) @classmethod def saveSimScratch(cls, df, date=None, name_without_path=None): """ hack method for auditing and analysis drop the df in the sim scratch directory for post inspection place a call whereever you want a dataframe for analysis, eg cls.saveSimScratch(dframe, tradeDate, "weights_early") args df: dataframe to save down date : if found appends to name name_without_path : no path, no extension """ user = os.environ['USER'] base = f"/home/{user}/scratch/r" os.makedirs(base, exist_ok=True) if name_without_path is None: name_without_path = "scratch_file" if date: date_str = date.strftime("%Y%m%d") filebase = f"{base}/{name_without_path}_{date_str}" else: filebase = f"{base}/{name_without_path}" df.to_parquet(f"{filebase}.pq") df.to_csv(f"{filebase}.csv") @classmethod def run(cls, rebalConfig=RebalConfig_DEFAULT, signalDate=PyDate.asDate(20191115), preOptWeights=EMPTY_PORTFOLIO, NAV=100000000, solver=cp.SCS, maxIter=200000, cycleIter=5000, verbose=True, checkDCP=True, optimal=False, tradeRestrictions=None): tradeDate = PyDate.nextWeekday(signalDate) PyLog.info('optimizing for SD:{} / TD:{}'.format(PyDate.asISO(signalDate), PyDate.asISO(tradeDate))) dctOptResult = dict() dctOptResult['rebalConfig'] = rebalConfig dctOptResult['signalDate'] = signalDate dctOptResult['preOptNAV'] = NAV if len(preOptWeights) == 0: dfWeights = pd.DataFrame({'assetKey': ['assetKey'], 'preOptWeights': [0.0]}).iloc[:0] else: dfWeights = preOptWeights.reset_index().rename(columns={'index': 'assetKey', 0: 'preOptWeights'}) dfWeights = dfWeights[[Real.isNonZero(x) for x in dfWeights['preOptWeights']]] stratName = rebalConfig.stratName rmodel = SignalMgr.get(rebalConfig.rmodelName, signalDate, stratName) tmodel = SignalMgr.get(rebalConfig.tmodelName, signalDate, stratName).set_index('assetKey') # gmult = Trading.getModelParameters('rrmult', signalDate) PyLog.info("Rebalance.py.run SET GMULT=1.0 --FROM YB TRADING GOOGLE SHEET") gmult=1.0 PyLog.info("Rebalance.py.run SET TCAFMULT=1.0 --FROM YB TRADING GOOGLE SHEET") tcafmult =1.0 # tcafmult = Trading.getModelParameters('tcafmult', signalDate) PyLog.info(f"Enable enable_stock_selection_alpha is {rebalConfig.enable_stock_selection_alpha}") if rebalConfig.enable_stock_selection_alpha: signals = {rebalConfig.alphaName : 'alpha', 'stock_selection_alpha' : 'stock_selection_alpha', rebalConfig.fcostName : 'fcost', rebalConfig.bcostName : 'bcost', rebalConfig.longBoundName : 'longBoundSoft', rebalConfig.shortBoundName : 'shortBoundSoft', 'latest_mktcap_usdm' : 'mktcap', 'univwt' : 'univwt', 'model_country' : 'modelCountry'} else: signals = {rebalConfig.alphaName : 'alpha', rebalConfig.fcostName : 'fcost', rebalConfig.bcostName : 'bcost', rebalConfig.longBoundName : 'longBoundSoft', rebalConfig.shortBoundName : 'shortBoundSoft', 'latest_mktcap_usdm' : 'mktcap', 'univwt' : 'univwt', 'model_country' : 'modelCountry'} dframe = SignalMgr.getFrame(list(signals.keys()), signalDate, stratName).\ drop(columns=['signalDate']).rename(columns=signals).set_index('assetKey') PyLog.info(f"Optimzier countries = {rebalConfig.optimizer_countries}") ## filter dataframe by countries if rebalConfig.optimizer_countries is not None: dframe = dframe[dframe["modelCountry"].isin( rebalConfig.optimizer_countries)] tmult = SignalMgr.get('trading_tcost_amortization_factor_multiplier', signalDate, stratName).iloc[0] dct = {'tcafmult': tcafmult, 'tmult': tmult} ## hard/soft position bounds ------------------------------------------------------------------------------- strategy_type = rebalConfig.strategyLongShortType.upper() # Make sure shortBoundSoft exists so we can control it explicitly if "shortBoundSoft" not in dframe.columns: dframe["shortBoundSoft"] = 0.0 if strategy_type == "LONG": dframe["longBoundHard"] = 1.0 dframe["shortBoundHard"] = 0.0 dframe["shortBoundSoft"] = 0.0 PyLog.info( f"{signalDate} | strategyLongShortType={strategy_type} | " f"set longBoundHard=1 shortBoundHard=0 shortBoundSoft=0" ) elif strategy_type == "SHORT": dframe["longBoundHard"] = 0.0 dframe["shortBoundHard"] = 1.0 # India-specific fix: trading_ShortBounds_frame has IN soft short bound at 0, # so override to allow actual shorting for India short-only strategy if rebalConfig.optimizer_countries == ["IN"]: dframe["shortBoundSoft"] = 1.0 PyLog.info( f"{signalDate} | strategyLongShortType={strategy_type} | " f"set longBoundHard=0 shortBoundHard=1 shortBoundSoft=" f"{1.0 if rebalConfig.optimizer_countries == ['IN'] else 'as-is'}" ) elif strategy_type == "BOTH": dframe["longBoundHard"] = 1.0 dframe["shortBoundHard"] = 1.0 # India-specific fix: trading_bounds_frame has IN soft short bound at 0, # so override to allow actual shorting for India L/S strategy if rebalConfig.optimizer_countries == ["IN"]: dframe["shortBoundSoft"] = 1.0 PyLog.info( f"{signalDate} | strategyLongShortType={strategy_type} | " f"set longBoundHard=1 shortBoundHard=1 shortBoundSoft=" f"{1.0 if rebalConfig.optimizer_countries == ['IN'] else 'as-is'}" ) else: raise ValueError( f"Unknown strategyLongShortType={rebalConfig.strategyLongShortType}. " f"Expected one of LONG / SHORT / BOTH" ) dframe = dframe[dframe['univwt'] >= rebalConfig.univwtThreshold] dframe = dframe[~dframe['alpha'].isnull()] # Apply stock selection alpha if rebalConfig.enable_stock_selection_alpha: dframe = apply_stock_selection_alpha(rebalConfig=rebalConfig, dframe=dframe, signalDate=signalDate) PyLog.info(f"Enable enable_simple_alpha is {rebalConfig.enable_simple_alpha}") # Apply simple alpha if rebalConfig.enable_simple_alpha: dframe = apply_simple_alpha(rebalConfig=rebalConfig, dframe=dframe, signalDate=signalDate) PyLog.info(f"Enable enable_3ai_alpha is {rebalConfig.enable_3ai_alpha}") # Apply 3ai alpha if rebalConfig.enable_3ai_alpha: PyLog.info( f"Applying 3AI alpha | " f"three_ai_alpha_countries={rebalConfig.three_ai_alpha_countries} | " f"strategyLongShortType={rebalConfig.strategyLongShortType} | " f"top_bottom_pct_3ai_stocks={rebalConfig.top_bottom_pct_3ai_stocks}" ) dframe = apply_3ai_alpha(rebalConfig=rebalConfig, dframe=dframe, signalDate=signalDate,) if rebalConfig.tradeUniverseName is not None: PyLog.info(f"Getting rebalConfig.tradeUniverseName:{rebalConfig.tradeUniverseName} signalDate:{signalDate} stratName:{stratName}") universe = SignalMgr.get(rebalConfig.tradeUniverseName, signalDate, stratName) dframe_sz_was = len(dframe) dframe = dframe[dframe.index.isin(universe.index)] dframe_sz_is = len(dframe) PyLog.info(f"dframe filtered from {dframe_sz_was} to {dframe_sz_is}. universe size:{len(universe)}") ## drop assets that are excluded from both long and short bounds dframe = dframe[~dframe['longBoundSoft'].isnull() | ~dframe['shortBoundSoft'].isnull()] # assets = list((set(rmodel['assets']).intersection(set(dframe.index)).intersection(set(tmodel.index))).\ # union(set(dfWeights['assetKey']))) assets = set(rmodel['assets']).intersection(set(dframe.index)) assets = assets.intersection(set(tmodel.index)) assets = sorted(list(assets)) dframe = dframe.reindex(index=assets).reset_index() dframe = dframe.assign(fcost = dframe['fcost'].fillna(rebalConfig.defaultFCost)) dframe = dframe.assign(bcost = dframe['bcost'].fillna(rebalConfig.defaultBCost)) ## position bounds ------------------------------------------------------------------------------------- dframe = dframe.assign(longBoundSoft = dframe['longBoundSoft'].fillna(rebalConfig.defaultLongBound)) dframe = dframe.assign(shortBoundSoft = dframe['shortBoundSoft'].fillna(rebalConfig.defaultShortBound)) PyLog.info(f"shortBoundSoft mean=" f"{dframe['shortBoundSoft'].mean():.4f}") if rebalConfig.strategyLongShortType != "LONG": ## in addition, we want to restrict short to 2% of market cap dframe = dframe.assign(shortAvail = dframe['mktcap'].fillna(10.0) * 0.02 * 1000000 / NAV) ## cut on min shortAvail in M;- i.e. zero out shortAvail if number is 0]) )) dframe.loc[dframe.shortAvail < minShortAvailM, 'shortAvail'] = 0 # dframe = dframe.assign(shortBound = dframe[['shortBound', 'shortAvail']].min(axis=1)) dframe = dframe.assign(shortBoundHard = dframe[['shortBoundHard', 'shortAvail']].min(axis=1)) dframe = dframe.drop(columns='shortAvail') else: PyLog.info( f"{signalDate} | strategyLongShortType={rebalConfig.strategyLongShortType} | " f"skipping shortAvail / minShortAvailM logic for LONG strategy" ) PyLog.info( f"{signalDate} | strategyLongShortType={rebalConfig.strategyLongShortType} | " f"maxLiquidityBoundLong={rebalConfig.maxLiquidityBoundLong} " f"maxLiquidityBoundShort={rebalConfig.maxLiquidityBoundShort}" ) ## apply liquidity-based position bounds --------------------------------------------------------------- if (rebalConfig.maxLiquidityBoundLong is not None) or (rebalConfig.maxLiquidityBoundShort is not None): dfm = SignalMgr.getFrame(rebalConfig.liquidityName, signalDate, stratName) dframe = dframe.merge(dfm[['assetKey', rebalConfig.liquidityName]], how='left', on='assetKey') dframe[rebalConfig.liquidityName] = \ dframe[rebalConfig.liquidityName].fillna(rebalConfig.defaultLiquidityUSDM) if rebalConfig.maxLiquidityBoundLong is not None: dframe = dframe.assign( liquidityBoundLong = rebalConfig.maxLiquidityBoundLong * dframe[rebalConfig.liquidityName] * 1000000 / NAV) dframe = dframe.assign(longBoundSoft = dframe[['longBoundSoft', 'liquidityBoundLong']].min(axis=1)) dframe = dframe.drop(columns=['liquidityBoundLong']) if rebalConfig.maxLiquidityBoundShort is not None: dframe = dframe.assign( liquidityBoundShort = rebalConfig.maxLiquidityBoundShort * dframe[rebalConfig.liquidityName] * 1000000 / NAV) dframe = dframe.assign(shortBoundSoft = dframe[['shortBoundSoft', 'liquidityBoundShort']].min(axis=1)) PyLog.info(f"shortBoundSoft mean=" f"{dframe['shortBoundSoft'].mean():.4f}") dframe = dframe.drop(columns=['liquidityBoundShort']) dframe = dframe.drop(columns=[rebalConfig.liquidityName]) ## ----------------------------------------------------------------------------------------------------- ## apply short bans ------------------------------------------------------------------------------------ if rebalConfig.applyShortBan: if PyDate.ge(signalDate, PyMonth.firstWeekday(202003)): if PyDate.le(signalDate, PyMonth.firstWeekday(202105)): countries = ['KR', 'ID'] else: countries = ['ID'] dframe = dframe.assign( preOptWeights = Filter.bound(-preOptWeights.reindex(dframe['assetKey']).fillna(0.0), lower=0.0)) dframe.loc[dframe['modelCountry'].isin(countries), 'preOptWeights'] =\ dframe.loc[dframe['modelCountry'].isin(countries), 'preOptWeights'] dframe.loc[~dframe['modelCountry'].isin(countries), 'preOptWeights'] = 1.0 # dframe = dframe.assign(shortBound = dframe[['shortBound', 'preOptWeights']].min(axis=1)) dframe = dframe.assign(shortBoundHard = dframe[['shortBoundHard', 'preOptWeights']].min(axis=1)) dframe = dframe.drop(columns='preOptWeights') ## ----------------------------------------------------------------------------------------------------- PyLog.info(f"dframe len:{len(dframe)} {dframe.columns.tolist()}") ######## Go impose TradeRestrictions pre-optimiser on longBoundHard/ShortBoundHard if(tradeRestrictions): from .TradeRestrictions import TradeRestrictions added_preOptWeights=False if not 'preOptWeights' in dframe.columns: preOptWeights_df = pd.DataFrame({'assetKey':preOptWeights.index, 'preOptWeights':preOptWeights.values}) dframe = dframe.merge(preOptWeights_df, how='left', on='assetKey').fillna(0.0) added_preOptWeights=True dframe = tradeRestrictions.applyTradeRestrictions( tradeDate, dframe, NAV) if added_preOptWeights and 'preOptWeights' in dframe.columns: dframe.drop(['preOptWeights'], axis=1, inplace=True) # ############### Done impose TradeRestrictions ###################################################### ## dframe is what goes into optimization / dfWeights is for record keeping dframe = dframe.merge(dfWeights, how='left', on='assetKey') ################ DO NOT TRADE PERISCOPE STOCKS AND ALSO UNWIND EXISTING PERISCOPE STOCKS #### periscope_df = EG.read_periscope_current_stocks() if periscope_df is None or periscope_df.empty: PyLog.info(f"PERISCOPE: no stocks for optimizer for {tradeDate}") periscope_keys = set() else: periscope_keys = set(periscope_df['assetKey']) PyLog.info(f"PERISCOPE: {len(periscope_keys)} stocks loaded for optimizer for {tradeDate}") is_periscope = dframe['assetKey'].isin(periscope_keys) pre = dframe['preOptWeights'].fillna(0.0) is_active = abs(pre) > 1e-6 # longs → force reduce to 0 dframe.loc[is_periscope & (pre > 0), 'longBoundHard'] = 0.0 # shorts → force cover to 0 dframe.loc[is_periscope & (pre < 0), 'shortBoundHard'] = 0.0 n_long = ((is_periscope) & (pre > 0)).sum() n_short = ((is_periscope) & (pre < 0)).sum() PyLog.info(f"PERISCOPE UNWIND: {n_long} long, {n_short} short positions forced to unwind for {tradeDate}") df_unwind = dframe[is_periscope & is_active][['assetKey', 'preOptWeights', 'longBoundHard', 'shortBoundHard']] PyLog.info(f"PERISCOPE UNWIND DETAIL ({len(df_unwind)} rows):") PyLog.info(df_unwind) # block NEW trades (only for flat positions) dframe.loc[is_periscope & (~is_active), 'longBoundHard'] = 0.0 dframe.loc[is_periscope & (~is_active), 'shortBoundHard'] = 0.0 n_block = ((is_periscope) & (~is_active)).sum() PyLog.info(f"PERISCOPE BLOCK: {n_block} stocks blocked from opening new positions") df_block = dframe[is_periscope & (~is_active)][['assetKey', 'preOptWeights', 'longBoundHard', 'shortBoundHard']] PyLog.info(f"PERISCOPE BLOCK DETAIL ({len(df_block)} rows):") PyLog.info(df_block) ###### END OF PERISCOPE UNWIND ####################### """ ################# ADD PERISCOPE ALPHA ####################### PyLog.info(f"Before apply periscope overlay") # Periscope overlay (updates dframe hard bounds before any splitLongShort processing) if str(getattr(rebalConfig, "periscopeMode", "OFF")).upper() != "OFF": applyPeriscopeOverlay(rebalConfig, signalDate, tradeDate, dframe, assets, preOptWeights) PyLog.info(f"After apply periscope overlay") ################# ADD PERISCOPE ALPHA ####################### """ ################ ADD TRBC Activities to be used later in the code ############## import model.utils.ric_tracer as rictr dframe = rictr.classify_rics_by_trbc(dframe) ################ ADD TRBC Activities to be used later in the code ############## PyLog.info(f"tradeDate: {tradeDate} rebalConfig.splitLongShort:{rebalConfig.splitLongShort} rebalConfig.splitLongShortStartDate:{rebalConfig.splitLongShortStartDate}") ## Note, the time condition here is different from that in Simulation. We want the splitLongShort ## applied in optimization first before applying it in the portfolio accounting. if rebalConfig.splitLongShort and (tradeDate >= rebalConfig.splitLongShortStartDate): dfAvail, dfMarginal = cls.compileShortAvailability(rebalConfig, NAV, tradeDate, dframe) #this was allowing optimal starting positions to be constructed exceeding avail borrow. Commenting out. #if optimal: # dfMarginal = dfAvail.groupby('assetKey').agg({'shortBoundHard': sum}).reset_index().\ # rename(columns={'shortBoundHard': 'availablePct'}) #dframe = dframe.merge(dfMarginal.rename(columns={'availablePct': 'shortBound'}), how='left', on='assetKey') dframe = dframe.merge(dfMarginal.rename(columns={'availablePct': 'shortBound'}), how='left', on='assetKey') dframe = dframe.assign(shortBound=Real.isNegative(dframe['preOptWeights']).astype(int) * abs(dframe['preOptWeights'].fillna(0.0)) + dframe['shortBound'].fillna(0.0) + rebalConfig.holdingThreshold / 100.0) dframe = dframe.assign(shortBoundHard=dframe[['shortBoundHard', 'shortBound']].min(axis=1)) """ ## reflect exchange holidays if rebalConfig.applyExchangeHolidays and (not optimal): lstHolidays = cls.getExchangeHolidays(tradeDate) dframe = dframe.assign(holiday=dframe['modelCountry'].isin(lstHolidays)) dframe.loc[dframe['holiday'], 'longBoundHard'] \ = Filter.bound(dframe.loc[dframe['holiday'], 'preOptWeights'].fillna(0.0), lower=0.0) \ + rebalConfig.holdingThreshold / 100.0 dframe.loc[dframe['holiday'], 'shortBoundHard'] \ = Filter.bound(-dframe.loc[dframe['holiday'], 'preOptWeights'].fillna(0.0), lower=0.0) \ + rebalConfig.holdingThreshold / 100.0 ## reflect exchange holidays """ if 'holiday' not in dframe.columns: dframe = dframe.assign(holiday=False) ## NEW reflect exchange holidays if rebalConfig.applyExchangeHolidays and (not optimal): try: lstHolidays = cls.getExchangeHolidays(tradeDate) PyLog.info(f"tradeDate = {tradeDate}, lstHolidays={lstHolidays}") dframe = dframe.assign(holiday=dframe['modelCountry'].isin(lstHolidays)) PyLog.info("===== HOLIDAY DEBUG: COUNTRY MATCH =====") PyLog.info(f"TradeDate: {tradeDate}") PyLog.info(f"Exchange holidays (lstHolidays): {sorted(list(lstHolidays))}") PyLog.info( "modelCountry counts:\n" + dframe['modelCountry'].value_counts(dropna=False).to_string() ) PyLog.info( "Detected holiday assets by modelCountry:\n" + dframe.loc[dframe['holiday'], 'modelCountry'] .value_counts() .to_string() ) PyLog.info( f"Total holiday assets detected: {int(dframe['holiday'].sum())} / {len(dframe)}" ) cur = dframe['preOptWeights'].fillna(0.0) cur_long = Filter.bound(cur, lower=0.0) cur_short = Filter.bound(-cur, lower=0.0) dframe.loc[dframe['holiday'], 'longBoundHard'] = cur_long.loc[dframe['holiday']] dframe.loc[dframe['holiday'], 'shortBoundHard'] = cur_short.loc[dframe['holiday']] except Exception as e: import traceback PyLog.error(f"Exception inside holiday block: {type(e).__name__}: {e}") PyLog.error(traceback.format_exc()) raise ## NEW reflect exchange holidays ## NEW reflect exchange holidays in dfAvail also dframe = dframe.assign(idx = list(range(len(dframe)))) dfAvail = dfAvail.merge(dframe[['assetKey', 'idx']], how='left', on='assetKey') if rebalConfig.applyExchangeHolidays and (not optimal): lk = dframe.set_index('assetKey') # update only the holiday rows in dfAvail hol_keys = set(dframe.loc[dframe['holiday'], 'assetKey']) mask_av_hol = dfAvail['assetKey'].isin(hol_keys) if 'longBoundHard' in dframe.columns: dfAvail.loc[mask_av_hol, 'longBoundHard'] = dfAvail.loc[mask_av_hol, 'assetKey'].map(lk['longBoundHard']) if 'shortBoundHard' in dframe.columns: dfAvail.loc[mask_av_hol, 'shortBoundHard'] = dfAvail.loc[mask_av_hol, 'assetKey'].map(lk['shortBoundHard']) PyLog.info(f"Unique modelCountry values: {dframe['modelCountry'].unique()[:10]}") PyLog.info(f"lstHolidays: {lstHolidays}") # Debugging: verify holiday logic and propagation num_holidays = dframe['holiday'].sum() if 'holiday' in dframe.columns else 0 PyLog.info(f"Detected {num_holidays} holiday assets out of {len(dframe)}") if num_holidays > 0: PyLog.info("Sample holiday-pinned assets (up to 5 per country):") holidays_df = dframe.loc[dframe['holiday']] for ctry, sub in holidays_df.groupby('modelCountry'): sample = sub[['assetKey']].head(5) PyLog.info(f"\nCountry: {ctry} — {len(sub)} total holiday assets, showing up to 5 assetKeys:\n" + sample.to_string(index=False)) # Cross-check dfAvail bounds match for those same assets merged_check = dfAvail.merge( dframe.loc[dframe['holiday'], ['assetKey', 'longBoundHard', 'shortBoundHard']], how='inner', on='assetKey', suffixes=('_avail', '_dframe') ) diff_long = (merged_check['longBoundHard_avail'] - merged_check['longBoundHard_dframe']).abs().sum() diff_short = (merged_check['shortBoundHard_avail'] - merged_check['shortBoundHard_dframe']).abs().sum() PyLog.info(f"Holiday bounds sync check — long diff: {diff_long:.3e}, short diff: {diff_short:.3e}") ## NEW reflect exchange holidays in dfAvail also """ ############### High-volatility TRBC Activity hard bounds (unwind only, no new risk) ############ dframe, dfAvail, activities_found = apply_high_vol_activity_hard_bounds( dframe, dfAvail=dfAvail, activity_col="Activity", vol_col="annualized", vol_abs_threshold=0.8, tradeDate=tradeDate, ) if activities_found == True: n_frozen = (dframe["longBoundHard"] == 0).sum() + (dframe["shortBoundHard"] == 0).sum() PyLog.info(f"High-vol Activity freeze: updated hard bounds for {n_frozen} (long/short) entries") else: PyLog.info("High-vol Activity freeze skipped: no activities above vol threshold") ############### High-volatility TRBC Activity hard bounds (unwind only, no new risk) ############ """ availAggr = np.zeros((len(dframe), len(dfAvail))) for n in range(len(dfAvail)): availAggr[dfAvail.iloc[n]['idx'], n] = 1 factors = rmodel['factors'] fload = rmodel['fload'].reindex(index=assets, columns=factors) ###################### Trend + Vol Conditioning PyLog.info(f"Applying alpha conditioning {rebalConfig.enable_alpha_conditioning}") if rebalConfig.enable_alpha_conditioning: PyLog.info("Before Applying trend+vol conditioning ") dframe = apply_alpha_conditioning(dframe, signalDate, rebalConfig) PyLog.info("After Conditioning complete") ###################### Trend + Vol Conditioning mu = dframe[['alpha']].to_numpy() D = np.diag(rmodel['srisk'].reindex(index=assets) ** 2) ## optional off-diagonal risk if rebalConfig.offDiagRisk: scov = rmodel['scov'] scov = scov.merge(dframe[['assetKey', 'idx']].rename(columns={'assetKey': 'xKey', 'idx': 'xIdx'}), how='inner', on='xKey') scov = scov.merge(dframe[['assetKey', 'idx']].rename(columns={'assetKey': 'yKey', 'idx': 'yIdx'}), how='inner', on='yKey') for xIdx, yIdx, cov in zip(scov['xIdx'], scov['yIdx'], scov['cov']): D[xIdx, yIdx] = D[yIdx, xIdx] = cov ## optional off-diagonal risk S = rmodel['fcov'].reindex(index=factors, columns=factors).to_numpy() if rebalConfig.shrinkfcov: S = matrix_shrink(S) else: PyLog.info(f"rebalConfig.shrinkfcov:{rebalConfig.shrinkfcov} using fcov S as is") F = fload.to_numpy() w = cp.Variable((len(dfAvail), 1)) aggrw = availAggr @ w wlong = cp.pos(w) wshort = cp.neg(w) """ if getattr(rebalConfig, "periscopeMode", "OFF") == "SOFT_PENALTY": periscope_pen = buildPeriscopePenalty(rebalConfig, assets, aggrw) else: periscope_pen = None """ # China Connect short-side stability penalty (only if file exists and scores > 4) if rebalConfig.enable_gs_stability_score == True: PyLog.info(f"Looking at GS stability files for tradeDate = {tradeDate}") china_short_penalty = build_china_connect_short_penalty(dfAvail, wshort, tradeDate) else: PyLog.info(f"GS Stability flag set to {rebalConfig.enable_gs_stability_score} for tradeDate = {tradeDate}") f = F.T @ aggrw rtau = copy.copy(rebalConfig.tau) otau = tmult * rtau atau = cp.Parameter(nonneg=True) atau.value = max(0, otau - 1) btau = cp.Parameter(nonneg=True) btau.value = min(1, otau) Lmax = cp.Parameter() Lmax.value = rebalConfig.maxLeverage ## Lmax = rebalConfig.maxLeverage ## long financing cost / short borrow cost fcost = dfAvail['fcost'].to_numpy() @ wlong bcost = dfAvail['bcost'].to_numpy() @ wshort ## expected return / risk netReturn = mu.T @ aggrw - fcost - bcost variance = cp.quad_form(f, S) + cp.quad_form(aggrw, D) ## transactions cost tmodel = tmodel.reindex(index=assets) trade = (aggrw - preOptWeights.reindex(index=assets).fillna(0.0).to_frame().to_numpy()) tbuy = cp.pos(trade) tsell = cp.neg(trade) linearCost = tmodel['linearBuyCoeff'].to_numpy() @ tbuy + tmodel['linearSellCoeff'].to_numpy() @ tsell impactBuy = cp.multiply(cp.sqrt(NAV) * rebalConfig.impactCostScale, tmodel['impactCoeffTH'].to_numpy() @ cp.power(tbuy, 3 / 2)) impactSell = cp.multiply(cp.sqrt(NAV) * rebalConfig.impactCostScale, tmodel['impactCoeffTH'].to_numpy() @ cp.power(tsell, 3 / 2)) tcost = linearCost + impactBuy + impactSell avgLinearCost = tmodel['linearAverageCoeff'].to_numpy() @ (tbuy + tsell) avgTcost = avgLinearCost + impactBuy + impactSell ## constraints ------------------------------------------------------------------------------------ constraints = [] penalty = 0 # --- Turnover penalty (soft) ------------------------------------------------- # trade is already defined above as (aggrw - preOptWeights) or (w - preOptWeights) # lambda 0.01 means I need 30 bps of annualized alpha to justify 30% NAV of turnover # because lambda * turnover = 0.01 * 0.3, so lambda = 0.003 = 30bps and rhs of # equation is net annualized alpha (mu), so mu = 0.003 = 30bps turn_lambda = getattr(rebalConfig, "turnoverPenaltyLambda", 0.01) if turn_lambda > 0: # L1 turnover in *weights* (sum abs(delta_w)); dimensionless penalty = penalty + turn_lambda * cp.sum(cp.abs(trade)) PyLog.info(f"Turnover penalty enabled: lambda={turn_lambda}") # ----------------------------------------------------------------------------- """ if periscope_pen is not None: penalty = penalty + periscope_pen PyLog.info("Periscope alpha: SOFT_PENALTY added to objective") """ ## Freeze holiday assets: no trades on exchange holidays if rebalConfig.applyExchangeHolidays and (not optimal): try: if 'holiday' in dframe.columns and dframe['holiday'].any(): # preOptWeights in dframe order cur_vec = dframe['preOptWeights'].fillna(0.0).to_numpy().reshape(-1, 1) # indices in aggrw corresponding to holiday assets hol_idx = dframe.loc[dframe['holiday'], 'idx'].to_numpy() if len(hol_idx) > 0: # aggrw[hol_idx] == current weights → zero trades for those names constraints.append(aggrw[hol_idx] == cur_vec[hol_idx]) PyLog.info(f"Holiday freeze: pinned {len(hol_idx)} assets to preOptWeights.") except Exception as e: import traceback PyLog.error(f"Exception inside holiday-freeze block: {type(e).__name__}: {e}") PyLog.error(traceback.format_exc()) raise # Start with China Connect short penalty if present if rebalConfig.enable_gs_stability_score == True: PyLog.info("Adding china short penalty") if china_short_penalty is not None: penalty = penalty + china_short_penalty ## max leverage # constraints.append(cp.norm(w, 1) <= Lmax) # pen = cp.pos(cp.norm(w, 1) - Lmax - 0.1) pen = cp.pos(cp.norm(w, 1) - Lmax) penalty = penalty + pen ## min leverage # if rebalConfig.minLeverage is not None: # Lmin = cp.Parameter() # Lmin.value = rebalConfig.minLeverage # constraints.append(cp.norm(w, 1) >= Lmin) ## risk bound # constraints.append(variance <= (rebalConfig.maxRisk ** 2)) pen = 5 * cp.pos(variance - (rebalConfig.maxRisk ** 2)) penalty = penalty + pen ## net exposure bounds # constraints.append(cp.sum(w) >= rebalConfig.minNetExposure) # constraints.append(cp.sum(w) <= rebalConfig.maxNetExposure) """ pen = cp.pos(cp.sum(aggrw) - rebalConfig.maxNetExposure) \ + cp.pos(rebalConfig.minNetExposure - cp.sum(aggrw)) penalty = penalty + pen """ PyLog.info(f"minNetExposure={rebalConfig.minNetExposure} maxNetExposure={rebalConfig.maxNetExposure}") ## net exposure bounds ------------------------------------------------------------- if rebalConfig.strategyLongShortType == "LONG": pen = cp.pos(cp.sum(aggrw) - rebalConfig.longOnlyMaxNetExposure) \ + cp.pos(rebalConfig.longOnlyMinNetExposure - cp.sum(aggrw)) penalty = penalty + pen elif rebalConfig.strategyLongShortType == "SHORT": # placeholder for future short-only configs pen = cp.pos(cp.sum(aggrw) - rebalConfig.maxNetExposure) \ + cp.pos(rebalConfig.minNetExposure - cp.sum(aggrw)) penalty = penalty + pen else: # BOTH = existing long/short behavior pen = cp.pos(cp.sum(aggrw) - rebalConfig.maxNetExposure) \ + cp.pos(rebalConfig.minNetExposure - cp.sum(aggrw)) penalty = penalty + pen ## individual position bounds - hard --------------------------------------------------------- constraints.append(cp.max(availAggr @ wlong - dframe[['longBoundHard']].to_numpy()) <= 0.0) constraints.append(cp.max(availAggr @ wshort - dframe[['shortBoundHard']].to_numpy()) <= 0.0) constraints.append(cp.max(wlong - dfAvail[['longBoundHard']].to_numpy()) <= 0.0) constraints.append(cp.max(wshort - dfAvail[['shortBoundHard']].to_numpy()) <= 0.0) ## individual position bounds - soft --------------------------------------------------------- pen = cp.sum(cp.pos(cp.pos(aggrw) - dframe[['longBoundSoft']].to_numpy())) \ + cp.sum(cp.pos(cp.neg(aggrw) - dframe[['shortBoundSoft']].to_numpy())) penalty = penalty + pen ## apply combined position bounds =========================================================== if rebalConfig.offDiagRisk: # if rebalConfig.applyExchangeHolidays and not 'HK' in lstHolidays: combAggr = np.zeros((len(scov), len(assets))) for n in range(len(scov)): combAggr[n, scov['xIdx'].iloc[n]] = combAggr[n, scov['yIdx'].iloc[n]] = 1 dfm = scov.merge(dframe[['assetKey', 'longBoundSoft', 'shortBoundSoft']]. rename( columns={'assetKey': 'xKey', 'longBoundSoft': 'xLong', 'shortBoundSoft': 'xShort'}), how='left', on='xKey') dfm = dfm.merge(dframe[['assetKey', 'longBoundSoft', 'shortBoundSoft']]. rename( columns={'assetKey': 'yKey', 'longBoundSoft': 'yLong', 'shortBoundSoft': 'yShort'}), how='left', on='yKey') dfm = dfm.assign(longBoundSoft=1.05 * dfm[['xLong', 'yLong']].max(axis=1)) dfm = dfm.assign(shortBoundSoft=1.05 * dfm[['xShort', 'yShort']].max(axis=1)) PyLog.info(f"shortBoundSoft mean=" f"{dframe['shortBoundSoft'].mean():.4f}") # dfm = dfm.assign(grossBound = 1.5 * dfm[['xLong', 'yLong', 'xShort', 'yShort']].max(axis=1)) if len(dfm) > 0: # constraints.append(cp.max(cp.pos(aggr @ w) - dfm[['longBound']].to_numpy()) <= 0.0) # constraints.append(cp.max(cp.neg(aggr @ w) - dfm[['shortBound']].to_numpy()) <= 0.0) pen = cp.sum(cp.pos(cp.pos(combAggr @ aggrw) - dfm[['longBoundSoft']].to_numpy())) \ + cp.sum(cp.pos(cp.neg(combAggr @ aggrw) - dfm[['shortBoundSoft']].to_numpy())) penalty = penalty + pen ## ========================================================================================== ## country net/gross exposures #if not Strategy.isSingleCountry(stratName): # apply country gross and net penalities only if we have more than 1 country in the dataframe if (rebalConfig.optimizer_countries is None or len(rebalConfig.optimizer_countries) != 1): cBounds = SignalMgr.get(rebalConfig.countryBoundName, signalDate, stratName) # if rebalConfig.applyExchangeHolidays: # cBounds = cBounds[~cBounds['cfactor'].isin(['country_' + x for x in lstHolidays])] cfactors = [x for x in factors if x.startswith('country_') and x in cBounds['cfactor'].tolist()] cBounds = cBounds.set_index('cfactor') cBounds = cBounds.reindex(index=cfactors) cBounds = cBounds.assign(maxGross=cBounds['maxGross'].fillna(0.0)) cBounds = cBounds.assign(minNet=cBounds['minNet'].fillna(0.0)) cBounds = cBounds.assign(maxNet=cBounds['maxNet'].fillna(0.0)) if rebalConfig.tightCountry: PyLog.info(f"tightening country bounds rebalConfig.tightCountry: {rebalConfig.tightCountry} stratName:{stratName}") cBounds['minNet'] = -0.0 cBounds['maxNet'] = 0.0 #PyLog.info(f"these cBounds are\n{cBounds} total max gross {cBounds.maxGross.sum() :.4f} ") # china hack. lower china leave room for JP, TH cBounds.loc["country_CN", "maxGross"] = 0.34 cBounds.loc["country_JP", "maxGross"] = 0.40 # change cbounds for thanilan to 2.5% of gross #cBounds.loc["country_TH", "maxGross"] = 0.16 cBounds.loc["country_TH", "maxGross"] = 0.025 #PyLog.info(f"after setting china these cBounds are\n{cBounds} total max gross {cBounds.maxGross.sum() :.4f} ") cNetExp = fload.reindex(index=assets, columns=cfactors).to_numpy().T @ aggrw maxNet = cBounds[['maxNet']].to_numpy().reshape(cNetExp.shape) minNet = cBounds[['minNet']].to_numpy().reshape(cNetExp.shape) #constraints.append(cp.max(cNetExp - maxNet) <= 0.0) #constraints.append(cp.min(cNetExp - minNet) >= 0.0) if rebalConfig.strategyLongShortType != "LONG": if rebalConfig.tightCountry: K = 50 PyLog.info(f"tightening country bounds with high penalty {K} rebalConfig.tightCountry: {rebalConfig.tightCountry} stratName:{stratName}") pen = K * cp.sum(cp.pos(cNetExp - maxNet)) + K * cp.sum(cp.pos(minNet - cNetExp)) else: pen = cp.sum(cp.pos(cNetExp - maxNet)) + cp.sum(cp.pos(minNet - cNetExp)) penalty = penalty + pen PyLog.info(f"Applying country Bounds for signal date {signalDate} for optimizer_countries {rebalConfig.optimizer_countries}") else: PyLog.info(f"{signalDate} | strategyLongShortType=LONG | skipping country NET penalties") cGrossExp = fload.reindex(index=assets, columns=cfactors).to_numpy().T @ cp.abs(aggrw) # maxGross = (rebalConfig.maxLeverage * 1.05 * cBounds[['maxGross']]).to_numpy().reshape(cGrossExp.shape) maxGross = (rebalConfig.maxLeverage * cBounds[['maxGross']]).to_numpy().reshape(cGrossExp.shape) # constraints.append(cp.max(cGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(cGrossExp - maxGross)) penalty = penalty + pen ## industry net/gross exposures iBounds = SignalMgr.get(rebalConfig.industryBoundName, signalDate, stratName) #PyLog.info(f"iBounds are: {iBounds}") ifactors = [x for x in factors if x.startswith('ind_') if x in iBounds['ifactor'].tolist()] iBounds = iBounds.set_index('ifactor') iBounds = iBounds.reindex(index=ifactors) iBounds = iBounds.assign(maxGross=iBounds['maxGross'].fillna(0.0)) iBounds = iBounds.assign(minNet=iBounds['minNet'].fillna(0.0)) iBounds = iBounds.assign(maxNet=iBounds['maxNet'].fillna(0.0)) iNetExp = fload.reindex(index=assets, columns=ifactors).to_numpy().T @ aggrw maxNet = iBounds[['maxNet']].to_numpy().reshape(iNetExp.shape) minNet = iBounds[['minNet']].to_numpy().reshape(iNetExp.shape) # constraints.append(cp.max(iNetExp - maxNet) <= 0.0) # constraints.append(cp.min(iNetExp - minNet) >= 0.0) if rebalConfig.strategyLongShortType != "LONG": PyLog.info(f"Applying industry bounds for signal date {signalDate}") pen = cp.sum(cp.pos(iNetExp - maxNet)) + cp.sum(cp.pos(minNet - iNetExp)) penalty = penalty + pen else: PyLog.info(f"{signalDate} | strategyLongShortType=LONG | skipping industry NET penalties") iGrossExp = fload.reindex(index=assets, columns=ifactors).to_numpy().T @ cp.abs(aggrw) # maxGross = (rebalConfig.maxLeverage * 1.05 * iBounds[['maxGross']]).to_numpy().reshape(iGrossExp.shape) maxGross = (rebalConfig.maxLeverage * iBounds[['maxGross']]).to_numpy().reshape(iGrossExp.shape) # constraints.append(cp.max(iGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(iGrossExp - maxGross)) penalty = penalty + pen ## size net/gross exposures sfactors = [x for x in factors if x.startswith('size_')] sBounds = SignalMgr.get(rebalConfig.sizeBoundName, signalDate, stratName).set_index('sfactor') sBounds = sBounds.reindex(index=sfactors) sBounds = sBounds.assign(maxGross=sBounds['maxGross'].fillna(0.0)) sBounds = sBounds.assign(minNet=sBounds['minNet'].fillna(0.0)) sBounds = sBounds.assign(maxNet=sBounds['maxNet'].fillna(0.0)) sNetExp = fload.reindex(index=assets, columns=sfactors).to_numpy().T @ aggrw maxNet = sBounds[['maxNet']].to_numpy().reshape(sNetExp.shape) minNet = sBounds[['minNet']].to_numpy().reshape(sNetExp.shape) # constraints.append(cp.max(sNetExp - maxNet) <= 0.0) # constraints.append(cp.min(sNetExp - minNet) >= 0.0) if rebalConfig.strategyLongShortType != "LONG": PyLog.info(f"Applying size bounds for signal date {signalDate}") pen = cp.sum(cp.pos(sNetExp - maxNet)) + cp.sum(cp.pos(minNet - sNetExp)) penalty = penalty + pen else: PyLog.info(f"{signalDate} | strategyLongShortType=LONG | skipping size NET penalties") sGrossExp = fload.reindex(index=assets, columns=sfactors).to_numpy().T @ cp.abs(aggrw) # maxGross = (rebalConfig.maxLeverage * 1.05 * sBounds[['maxGross']]).to_numpy().reshape(sGrossExp.shape) maxGross = (rebalConfig.maxLeverage * sBounds[['maxGross']]).to_numpy().reshape(sGrossExp.shape) sbounds_maxGross = sBounds[['maxGross']] PyLog.info(f"maxGross: {maxGross} = rebalConfig.maxLeverage {rebalConfig.maxLeverage} * {sbounds_maxGross}") # constraints.append(cp.max(sGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(sGrossExp - maxGross)) penalty = penalty + pen """ ## net exposure to sbeta_market mNetExp = fload.reindex(index=assets, columns=['sbeta_market']).to_numpy().T @ aggrw constraints.append(cp.min(mNetExp) >= rebalConfig.minBetaAdjNet) constraints.append(cp.max(mNetExp) <= rebalConfig.maxBetaAdjNet) """ ## net exposure to sbeta_market mNetExp = fload.reindex(index=assets, columns=['sbeta_market']).to_numpy().T @ aggrw strategy_type = rebalConfig.strategyLongShortType.upper() if strategy_type == "LONG": min_beta_adj_net = rebalConfig.longOnlyMinBetaAdjNet max_beta_adj_net = rebalConfig.longOnlyMaxBetaAdjNet else: min_beta_adj_net = rebalConfig.minBetaAdjNet max_beta_adj_net = rebalConfig.maxBetaAdjNet PyLog.info( f"mNetExp: {mNetExp} " f"strategyLongShortType={strategy_type} " f"minBetaAdjNet={min_beta_adj_net} " f"maxBetaAdjNet={max_beta_adj_net}" ) constraints.append(cp.min(mNetExp) >= min_beta_adj_net) constraints.append(cp.max(mNetExp) <= max_beta_adj_net) #PyLog.info(f"mNetExp: {mNetExp} rebalConfig.minBetaAdjNet:{rebalConfig.minBetaAdjNet} rebalConfig.maxBetaAdjNet:{rebalConfig.maxBetaAdjNet}") ## phase out value_liq in 2017~ =========================================================== # if PyDate.ge(signalDate, 20170101) and PyDate.le(signalDate, PyMonth.lastWeekday(202012)): # # if PyDate.ge(signalDate, 20170101): # vliqExp = fload.reindex(index=assets, columns=['value_liq']).to_numpy().T @ w # bound = 2.0 * Filter.bound(PyDate.span(signalDate, 20170331) / 90, lower=0.01) # constraints.append(cp.max(vliqExp) <= bound) # constraints.append(cp.min(vliqExp) >= -bound) ## ========================================================================================== # mfm = SignalMgr.getStatic('model_universe_frame') mfm = mfm.rename(columns={'RkdTicker': 'ticker', 'RkdDisplayRIC': 'ric', 'Country': 'quoteCountry'}) dframe = mfm[['assetKey', 'ticker', 'quoteCountry', 'ric']].merge(dframe, how='right', on='assetKey') ## apply alpha theme exposure bounds ======================================================== PyLog.info(f"Theme bound name is {rebalConfig.themeBoundName} for {signalDate}") if rebalConfig.themeBoundName is not None: themeBounds = SignalMgr.get(rebalConfig.themeBoundName, signalDate, stratName) themeBounds = FrameUtil.toSeries(themeBounds, keyCol='factor', valCol='bound') * rebalConfig.maxLeverage themes = list(themeBounds.index) themeExp = fload.reindex(index=assets, columns=themes).to_numpy().T @ aggrw # constraints.append(cp.max(themeExp - themeBounds.to_numpy().reshape(themeExp.shape)) <= 0) # constraints.append(cp.min(themeExp + themeBounds.to_numpy().reshape(themeExp.shape)) >= 0) pen = cp.sum(cp.pos(themeExp - themeBounds.to_numpy().reshape(themeExp.shape))) \ + cp.sum(cp.pos(- themeExp - themeBounds.to_numpy().reshape(themeExp.shape))) penalty = penalty + pen ## here we are constructing symmetric bounds around 0 but it is the upper bound that we are ## primarily concerned about ## ========================================================================================== ## penalty - apply max trade constraints ==================================================== if (not optimal) and (rebalConfig.maxAdvProp is not None): dfm = SignalMgr.getFrame(rebalConfig.advName, signalDate, stratName) dfm = dframe[['assetKey']].merge(dfm[['assetKey', rebalConfig.advName]], how='left', on='assetKey') dfm = dfm.assign( maxTradeWeights=rebalConfig.maxAdvProp * dfm[rebalConfig.advName].fillna(0.0) * 1000000 / NAV) ## may need to relax this condition for certain conditions (i.e. forced trading) # constraints.append(cp.max(cp.abs(trade) - dfm[['maxTradeWeights']].to_numpy()) <= 0.0) pen = (cp.abs(trade) - dfm[['maxTradeWeights']].to_numpy()) / dfm[['maxTradeWeights']].to_numpy() # pen = 4 * cp.max(cp.pos(pen)) pen = 4 * cp.sum(cp.pos(pen)) penalty = penalty + pen ## ========================================================================================== """ ################ THIS ENTIRE SECTION IS ADDING TRBC ACTIVITY LEVEL PENALTIES BASED ON SPECRET COV try: trbc_lambda = getattr(rebalConfig, "trbcClusterPenaltyLambda", 0.02) PyLog.info(f"TRBC lambda is {trbc_lambda}") trbc_mode = getattr(rebalConfig, "trbcClusterPenalizeBy", "PENALIZE_NET_EXPOSURE") trbc_pen, cluster_df = build_trbc_cluster_penalty( dframe=dframe, assets=assets, aggrw=aggrw, tradeDate=tradeDate, gqr_dir=GQR_SPECRET_DIR, corr_threshold=0.7, min_cluster_size=4, lambda_cluster=trbc_lambda, penalize_by=trbc_mode, ) if isinstance(trbc_pen, cp.Expression): penalty = penalty + trbc_pen PyLog.info("TRBC cluster penalty: added to objective") else: PyLog.info("TRBC cluster penalty: nothing to add (no clusters / files missing)") except Exception as e: PyLog.info(f"TRBC cluster penalty: failed to build ({type(e).__name__}: {e}) -> skipping") ################ END ADDING TRBC ACTIVITY LEVEL PENALTIES ############### """ gamma = cp.Parameter(nonneg=True) # Uncomment this to print final dataframe before optimization starts #dframe.to_csv("dframe_final.csv") PyLog.info(f"constraints==== {len(constraints)}===") PyLog.info(dframe[[ "shortBoundHard", "longBoundHard" ] ].describe() .to_string()) PyLog.info(dframe["alpha"].describe().to_string()) PyLog.info(f"nonzero alpha count=" f"{(dframe['alpha'].abs() > 1e-8).sum()}") PyLog.info(f"optimizer assets={len(dframe)}") PyLog.info(dframe["modelCountry"].value_counts().to_string()) PyLog.info(dframe["shortBoundSoft"].describe().to_string()) PyLog.info(f"Net exposure bounds: " f"{rebalConfig.minNetExposure:.2f} " f"to " f"{rebalConfig.maxNetExposure:.2f}") if rebalConfig.adaptiveGamma: prevGamma = 0.0 currGamma = 0.0 prevRisk = 0.0 currRisk = 0.0 offset = 1.0 while Real.isZero(currRisk) or \ abs(currRisk - rebalConfig.adaptiveGammaTargetRisk) > rebalConfig.adaptiveGammaTolerance: if Real.isZero(currRisk): rgamma = rebalConfig.gamma elif Real.isZero(prevRisk): rgamma = currGamma + np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) * offset elif Real.isPositive(np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) * np.sign(prevRisk - rebalConfig.adaptiveGammaTargetRisk)): rgamma = currGamma + np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) else: rgamma = (prevGamma + currGamma) / 2 offset = offset / 4 gamma.value = gmult * rgamma ## optimization setup if optimal: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - penalty), constraints) else: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - btau * tcost - atau * avgTcost - penalty), constraints) numIters = cls.solveProblem(signalDate, rebalConfig=rebalConfig, problem=problem, maxIter=maxIter, cycleIter=cycleIter, solver=solver, verbose=verbose, checkDCP=checkDCP, optimal=optimal) if w.value is None: PyLog.info(f"Optimizer failed to converge for {signalDate}") w.value = dfAvail[['preOptWeights']].fillna(0.0).to_numpy() weights = pd.Series(aggrw.value.flatten(), index=assets) prevGamma = currGamma prevRisk = currRisk currGamma = rgamma currRisk = RiskModel.computeRisk(rmodel, weights) else: rgamma = rebalConfig.gamma gamma.value = gmult * rgamma ## optimization setup if optimal: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - penalty), constraints) else: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - btau * tcost - atau * avgTcost - penalty), constraints) numIters = cls.solveProblem(signalDate, rebalConfig=rebalConfig, problem=problem, maxIter=maxIter, cycleIter=cycleIter, solver=solver, verbose=verbose, checkDCP=checkDCP, optimal=optimal) if w.value is None: PyLog.info(" Optimizer failed to converge") w.value = dfAvail[['preOptWeights']].fillna(0.0).to_numpy() optimizer_stats = { "gamma": float(gamma.value), "rtau": float(rtau), "tmult": float(tmult), "otau": float(otau), "atau": float(atau.value), "btau": float(btau.value), "objective": float(problem.value), "gross": float(np.sum(np.abs(aggrw.value))), "netReturn": float(netReturn.value), "variance": float(variance.value), "gamma_variance": float((gamma * variance).value), "linearCost": float(linearCost.value), "avgLinearCost": float(avgLinearCost.value), "impactBuy": float(impactBuy.value), "impactSell": float(impactSell.value), "tcost": float(tcost.value), "avgTcost": float(avgTcost.value), "btau_tcost": float((btau * tcost).value), "atau_avgTcost": float((atau * avgTcost).value), "penalty": float(penalty.value), "assets": len(assets), "NAV": float(NAV), } """ # Log TRBC cluster exposures and implied penalty log_trbc_cluster_exposures( tradeDate, assets, aggrw, cluster_df=cluster_df, gqr_dir=GQR_SPECRET_DIR, lambda_cluster=trbc_lambda) """ dfAvail = dfAvail.assign(optimalWeights = list(w.value.flatten())) dfAvail = dfAvail.assign(postOptWeights = dfAvail['optimalWeights']) # dfAvail.loc[abs(dfAvail['postOptWeights']) < rebalConfig.holdingThreshold, 'postOptWeights'] = 0.0 dframe = dframe.assign(optimalWeights = availAggr @ dfAvail['optimalWeights']) dframe = dframe.assign(postOptWeights = availAggr @ dfAvail['postOptWeights']) ########## Check if we have trades for holidays PyLog.info("===== STEP 1: HOLIDAY AGG TRADE CHECK =====") hol = dframe['holiday'].fillna(False) agg_trade = (dframe['postOptWeights'].fillna(0.0) - dframe['preOptWeights'].fillna(0.0)) PyLog.info(f"ABS SUM holiday AGG trade: {abs(agg_trade[hol]).sum():.12e}") bad = dframe.loc[hol].assign(trade=agg_trade[hol]).loc[lambda x: x['trade'].abs() > 1e-5] PyLog.info(f"Holiday assets with |agg trade|>1e-5: {len(bad)}") #### This is informational only because we are pinning postOptWeights = preOptWeights for holiday countries if len(bad): PyLog.info("Top holiday agg trades:\n" + bad[['assetKey','modelCountry','preOptWeights','postOptWeights','trade']] .head(20).to_string()) PyLog.info("===== STEP 2: DFAVAIL MULTI-ROW CHECK (HOLIDAYS) =====") hol_keys = set(dframe.loc[dframe['holiday'].fillna(False), 'assetKey']) cnt = dfAvail[dfAvail['assetKey'].isin(hol_keys)].groupby('assetKey').size() multi = cnt[cnt > 1].sort_values(ascending=False) PyLog.info(f"Holiday assetKeys with >1 dfAvail rows: {len(multi)}") if len(multi): PyLog.info("Top multi-row holiday assets:\n" + multi.head(30).to_string()) ########## Check if we have trades for holidays dfWeights = dfWeights.merge(dframe[['assetKey', 'optimalWeights', 'postOptWeights']], how='outer', on='assetKey') dfWeights.loc[abs(dfWeights['postOptWeights'].fillna(0.0)) < rebalConfig.holdingThreshold, 'postOptWeights'] = 0.0 dfWeights = dfWeights.assign( tradeWeights = dfWeights['postOptWeights'].fillna(0.0) - dfWeights['preOptWeights'].fillna(0.0)) PyLog.info(f"gross={np.sum(np.abs(aggrw.value)):.4f}") PyLog.info( f"net={np.sum(aggrw.value):.4f}") PyLog.info( f"long exposure=" f"{np.sum(np.maximum(aggrw.value, 0)):.4f}" ) PyLog.info( f"short exposure=" f"{-np.sum(np.minimum(aggrw.value, 0)):.4f}" ) ######### Do important post optimization validation and send email to support team ##### cls.post_optimization_validation( optimizer_stats, tradeDate=tradeDate, rebalConfig=rebalConfig, dfWeights=dfWeights, hard_fail=False, ) ######### Do important post optimization validation and send email to support team ##### # --- Start China penalty post-solve logging --- if rebalConfig.enable_gs_stability_score == True: if china_short_penalty is not None and hasattr(china_short_penalty, "_china_penalty_info"): try: info = china_short_penalty._china_penalty_info weights_arr = china_short_penalty._china_penalty_weights scale = china_short_penalty._china_scale PyLog.info("ChinaConnect: post-solve penalty details for penalized assets (top 50):") for idx, assetKey, score, pw in info[:50]: if wshort.value is None: short_w = None contrib = None else: short_w = float(wshort.value[idx]) contrib = scale * pw * short_w if short_w is not None else None PyLog.info( f" assetKey={assetKey}, score={score}, penaltyWeight={pw}, " f"shortWeight={(short_w if short_w is not None else 'None')}, " f"penaltyContribution={(contrib if contrib is not None else 'None')}" ) if len(info) > 50: PyLog.info(f" ...and {len(info) - 50} more penalized assets") total_pen = float(china_short_penalty.value) if china_short_penalty.value is not None else None PyLog.info(f"ChinaConnect: total China penalty = {total_pen}") except Exception as e: PyLog.error(f"ChinaConnect: error computing post-solve penalty log: {e}") # --- End China penalty post-solve logging --- # weights = pd.Series(w.value.flatten(), index=assets) ################################## ENTERTING USELESS SPLIT LONG SHORT BLOCK ######################## ################################## ENTERTING USELESS SPLIT LONG SHORT BLOCK ######################## ################################## ENTERTING USELESS SPLIT LONG SHORT BLOCK ######################## else: PyLog.info(f"Entering split block. rebalConfig.splitLongShort:{rebalConfig.splitLongShort} rebalConfig.splitLongShortStartDate:{rebalConfig.splitLongShortStartDate}") PyLog.info(f"rebalConfig.applyExchangeHolidays:{rebalConfig.applyExchangeHolidays} optimal:{optimal}") """ ## reflect exchange holidays ------------------------------------------------------------------- if rebalConfig.applyExchangeHolidays and (not optimal): lstHolidays = cls.getExchangeHolidays(tradeDate) dframe = dframe.assign(holiday=dframe['modelCountry'].isin(lstHolidays)) dframe.loc[dframe['holiday'], 'longBoundHard'] \ = Filter.bound(dframe.loc[dframe['holiday'], 'preOptWeights'].fillna(0.0), lower=0.0) \ + rebalConfig.holdingThreshold / 100.0 dframe.loc[dframe['holiday'], 'shortBoundHard'] \ = Filter.bound(-dframe.loc[dframe['holiday'], 'preOptWeights'].fillna(0.0), lower=0.0) \ + rebalConfig.holdingThreshold / 100.0 ## --------------------------------------------------------------------------------------------- """ ## NEW reflect exchange holidays if rebalConfig.applyExchangeHolidays and (not optimal): lstHolidays = cls.getExchangeHolidays(tradeDate) dframe = dframe.assign(holiday=dframe['modelCountry'].isin(lstHolidays)) cur = dframe['preOptWeights'].fillna(0.0) cur_long = Filter.bound(cur, lower=0.0) cur_short = Filter.bound(-cur, lower=0.0) dframe.loc[dframe['holiday'], 'longBoundHard'] = cur_long.loc[dframe['holiday']] dframe.loc[dframe['holiday'], 'shortBoundHard'] = cur_short.loc[dframe['holiday']] factors = rmodel['factors'] fload = rmodel['fload'].reindex(index=assets, columns=factors) ## NEW reflect exchange holidays mu = dframe[['alpha']].to_numpy() D = np.diag(rmodel['srisk'].reindex(index=assets) ** 2) ## optional off-diagonal risk ------------------------------------------------------------------------------ PyLog.info(f"rebalConfig.offDiagRisk:{rebalConfig.offDiagRisk}") if rebalConfig.offDiagRisk: scov = rmodel['scov'] dfm = pd.DataFrame({'assetKey': assets, 'idx': range(len(assets))}) scov = scov.merge(dfm.rename(columns={'assetKey': 'xKey', 'idx': 'xIdx'}), how='inner', on='xKey') scov = scov.merge(dfm.rename(columns={'assetKey': 'yKey', 'idx': 'yIdx'}), how='inner', on='yKey') for xIdx, yIdx, cov in zip(scov['xIdx'], scov['yIdx'], scov['cov']): D[xIdx, yIdx] = D[yIdx, xIdx] = cov ## --------------------------------------------------------------------------------------------------------- S = rmodel['fcov'].reindex(index=factors, columns=factors).to_numpy() if rebalConfig.shrinkfcov: S = matrix_shrink(S) else: PyLog.info(f"rebalConfig.shrinkfcov:{rebalConfig.shrinkfcov} using fcov S as is") F = fload.to_numpy() w = cp.Variable((len(assets), 1)) ## wlong = cp.maximum( w, 0) ## wshort = cp.maximum(-w, 0) wlong = cp.pos(w) wshort = cp.neg(w) f = F.T @ w rtau = rebalConfig.tau otau = tmult * rtau atau = cp.Parameter(nonneg=True) atau.value = max(0, otau - 1) btau = cp.Parameter(nonneg=True) btau.value = min(1, otau) Lmax = cp.Parameter() Lmax.value = rebalConfig.maxLeverage ## Lmax = rebalConfig.maxLeverage ## long financing cost / short borrow cost fcost = dframe['fcost'].to_numpy() @ wlong bcost = dframe['bcost'].to_numpy() @ wshort ## expected return / risk netReturn = mu.T @ w - fcost - bcost variance = cp.quad_form(f, S) + cp.quad_form(w, D) ## transactions cost tmodel = tmodel.reindex(index=assets) trade = (w - preOptWeights.reindex(index=assets).fillna(0.0).to_frame().to_numpy()) tbuy = cp.pos(trade) tsell = cp.neg(trade) linearCost = tmodel['linearBuyCoeff'].to_numpy() @ tbuy + tmodel['linearSellCoeff'].to_numpy() @ tsell ## impactCost = (tmodel['impactCoeffTH'].to_numpy() * ((NAV * (tbuy + tsell)) ** (3/2))) / NAV ## impactCost = tmodel['impactCoeffTH'].to_numpy() * cp.multiply(1/NAV, cp.power(cp.multiply(NAV, tbuy + tsell), 3/2)) ## impactBuy = cp.multiply(1 / NAV, tmodel['impactCoeffTH'].to_numpy() * cp.power(cp.multiply(NAV, tbuy), 3 / 2)) ## impactSell = cp.multiply(1 / NAV, tmodel['impactCoeffTH'].to_numpy() * cp.power(cp.multiply(NAV, tsell), 3 / 2)) impactBuy = cp.multiply(cp.sqrt(NAV) * rebalConfig.impactCostScale, tmodel['impactCoeffTH'].to_numpy() @ cp.power(tbuy, 3 / 2)) impactSell = cp.multiply(cp.sqrt(NAV) * rebalConfig.impactCostScale, tmodel['impactCoeffTH'].to_numpy() @ cp.power(tsell, 3 / 2)) tcost = linearCost + impactBuy + impactSell avgLinearCost = tmodel['linearAverageCoeff'].to_numpy() @ (tbuy + tsell) avgTcost = avgLinearCost + impactBuy + impactSell ## constraints -------------------------------------------------------------------- constraints = [] penalty = 0 ## max leverage # constraints.append(cp.norm(w, 1) <= Lmax) # pen = cp.pos(cp.norm(w, 1) - Lmax - 0.1) pen = cp.pos(cp.norm(w, 1) - Lmax) penalty = penalty + pen ## min leverage # if rebalConfig.minLeverage is not None: # Lmin = cp.Parameter() # Lmin.value = rebalConfig.minLeverage # constraints.append(cp.norm(w, 1) >= Lmin) ## risk bound # constraints.append(variance <= (rebalConfig.maxRisk ** 2)) pen = 5 * cp.pos(variance - (rebalConfig.maxRisk ** 2)) penalty = penalty + pen ## net exposure bounds # constraints.append(cp.sum(w) >= rebalConfig.minNetExposure) # constraints.append(cp.sum(w) <= rebalConfig.maxNetExposure) pen = cp.pos(cp.sum(w) - rebalConfig.maxNetExposure) \ + cp.pos(rebalConfig.minNetExposure - cp.sum(w)) penalty = penalty + pen ## individual position bounds - hard --------------------------------------------------------- constraints.append(cp.max(wlong - dframe[['longBoundHard']].to_numpy()) <= 0.0) constraints.append(cp.max(wshort - dframe[['shortBoundHard']].to_numpy()) <= 0.0) ## individual position bounds - soft --------------------------------------------------------- pen = cp.sum(cp.pos(cp.pos(w) - dframe[['longBoundSoft']].to_numpy())) \ + cp.sum(cp.pos(cp.neg(w) - dframe[['shortBoundSoft']].to_numpy())) penalty = penalty + pen ## apply combined position bounds =========================================================== if rebalConfig.offDiagRisk: # if rebalConfig.applyExchangeHolidays and not 'HK' in lstHolidays: aggr = np.zeros((len(scov), len(assets))) for n in range(len(scov)): aggr[n, scov['xIdx'].iloc[n]] = aggr[n, scov['yIdx'].iloc[n]] = 1 dfm = scov.merge(dframe[['assetKey', 'longBoundSoft', 'shortBoundSoft']]. rename(columns={'assetKey': 'xKey', 'longBoundSoft': 'xLong', 'shortBoundSoft': 'xShort'}), how='left', on='xKey') dfm = dfm.merge(dframe[['assetKey', 'longBoundSoft', 'shortBoundSoft']]. rename(columns={'assetKey': 'yKey', 'longBoundSoft': 'yLong', 'shortBoundSoft': 'yShort'}), how='left', on='yKey') dfm = dfm.assign(longBoundSoft = 1.05 * dfm[['xLong', 'yLong']].max(axis=1)) dfm = dfm.assign(shortBoundSoft = 1.05 * dfm[['xShort', 'yShort']].max(axis=1)) # dfm = dfm.assign(grossBound = 1.5 * dfm[['xLong', 'yLong', 'xShort', 'yShort']].max(axis=1)) if len(dfm) > 0: # constraints.append(cp.max(cp.pos(aggr @ w) - dfm[['longBound']].to_numpy()) <= 0.0) # constraints.append(cp.max(cp.neg(aggr @ w) - dfm[['shortBound']].to_numpy()) <= 0.0) pen = cp.sum(cp.pos(cp.pos(aggr @ w) - dfm[['longBoundSoft']].to_numpy())) \ + cp.sum(cp.pos(cp.neg(aggr @ w) - dfm[['shortBoundSoft']].to_numpy())) penalty = penalty + pen ## ========================================================================================== ## country net/gross exposures if not Strategy.isSingleCountry(stratName): cBounds = SignalMgr.get(rebalConfig.countryBoundName, signalDate, stratName) # if rebalConfig.applyExchangeHolidays: # cBounds = cBounds[~cBounds['cfactor'].isin(['country_' + x for x in lstHolidays])] cfactors = [x for x in factors if x.startswith('country_') and x in cBounds['cfactor'].tolist()] cBounds = cBounds.set_index('cfactor') cBounds = cBounds.reindex(index=cfactors) cBounds = cBounds.assign(maxGross = cBounds['maxGross'].fillna(0.0)) cBounds = cBounds.assign(minNet = cBounds['minNet'].fillna(0.0)) cBounds = cBounds.assign(maxNet = cBounds['maxNet'].fillna(0.0)) if rebalConfig.tightCountry: PyLog.info(f"tightening country bounds rebalConfig.tightCountry: {rebalConfig.tightCountry} stratName:{stratName}") cBounds['minNet'] = -0.0 cBounds['maxNet'] = 0.0 # aggrw is w cNetExp = fload.reindex(index=assets, columns=cfactors).to_numpy().T @ w maxNet = cBounds[['maxNet']].to_numpy().reshape(cNetExp.shape) minNet = cBounds[['minNet']].to_numpy().reshape(cNetExp.shape) PyLog.info(f"here cBounds are\n{cBounds}") #constraints.append(cp.max(cNetExp - maxNet) <= 0.0) #constraints.append(cp.min(cNetExp - minNet) >= 0.0) if rebalConfig.tightCountry: K = 50 PyLog.info(f"tightening country bounds with high penalty {K} rebalConfig.tightCountry: {rebalConfig.tightCountry} stratName:{stratName}") pen = K * cp.sum(cp.pos(cNetExp - maxNet)) + K * cp.sum(cp.pos(minNet - cNetExp)) else: pen = cp.sum(cp.pos(cNetExp - maxNet)) + cp.sum(cp.pos(minNet - cNetExp)) penalty = penalty + pen cGrossExp = fload.reindex(index=assets, columns=cfactors).to_numpy().T @ cp.abs(w) # maxGross = (rebalConfig.maxLeverage * 1.05 * cBounds[['maxGross']]).to_numpy().reshape(cGrossExp.shape) maxGross = (rebalConfig.maxLeverage * cBounds[['maxGross']]).to_numpy().reshape(cGrossExp.shape) # constraints.append(cp.max(cGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(cGrossExp - maxGross)) penalty = penalty + pen ## industry net/gross exposures iBounds = SignalMgr.get(rebalConfig.industryBoundName, signalDate, stratName) ifactors = [x for x in factors if x.startswith('ind_') if x in iBounds['ifactor'].tolist()] iBounds = iBounds.set_index('ifactor') iBounds = iBounds.reindex(index=ifactors) iBounds = iBounds.assign(maxGross = iBounds['maxGross'].fillna(0.0)) iBounds = iBounds.assign(minNet = iBounds['minNet'].fillna(0.0)) iBounds = iBounds.assign(maxNet = iBounds['maxNet'].fillna(0.0)) iNetExp = fload.reindex(index=assets, columns=ifactors).to_numpy().T @ w maxNet = iBounds[['maxNet']].to_numpy().reshape(iNetExp.shape) minNet = iBounds[['minNet']].to_numpy().reshape(iNetExp.shape) # constraints.append(cp.max(iNetExp - maxNet) <= 0.0) # constraints.append(cp.min(iNetExp - minNet) >= 0.0) pen = cp.sum(cp.pos(iNetExp - maxNet)) + cp.sum(cp.pos(minNet - iNetExp)) penalty = penalty + pen iGrossExp = fload.reindex(index=assets, columns=ifactors).to_numpy().T @ cp.abs(w) # maxGross = (rebalConfig.maxLeverage * 1.05 * iBounds[['maxGross']]).to_numpy().reshape(iGrossExp.shape) maxGross = (rebalConfig.maxLeverage * iBounds[['maxGross']]).to_numpy().reshape(iGrossExp.shape) # constraints.append(cp.max(iGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(iGrossExp - maxGross)) penalty = penalty + pen ## size net/gross exposures sfactors = [x for x in factors if x.startswith('size_')] sBounds = SignalMgr.get(rebalConfig.sizeBoundName, signalDate, stratName).set_index('sfactor') sBounds = sBounds.reindex(index=sfactors) sBounds = sBounds.assign(maxGross = sBounds['maxGross'].fillna(0.0)) sBounds = sBounds.assign(minNet = sBounds['minNet'].fillna(0.0)) sBounds = sBounds.assign(maxNet = sBounds['maxNet'].fillna(0.0)) sNetExp = fload.reindex(index=assets, columns=sfactors).to_numpy().T @ w maxNet = sBounds[['maxNet']].to_numpy().reshape(sNetExp.shape) minNet = sBounds[['minNet']].to_numpy().reshape(sNetExp.shape) # constraints.append(cp.max(sNetExp - maxNet) <= 0.0) # constraints.append(cp.min(sNetExp - minNet) >= 0.0) pen = cp.sum(cp.pos(sNetExp - maxNet)) + cp.sum(cp.pos(minNet - sNetExp)) penalty = penalty + pen PyLog.info(f"sfactors: {sfactors}") PyLog.info(f"fload:\n{fload[sfactors].iloc[0]}") sGrossExp = fload.reindex(index=assets, columns=sfactors).to_numpy().T @ cp.abs(w) # maxGross = (rebalConfig.maxLeverage * 1.05 * sBounds[['maxGross']]).to_numpy().reshape(sGrossExp.shape) maxGross = (rebalConfig.maxLeverage * sBounds[['maxGross']]).to_numpy().reshape(sGrossExp.shape) PyLog.info(f"sBounds are\n{sBounds}") sbounds_maxGross = sBounds[['maxGross']] PyLog.info(f"maxGross: {maxGross} = rebalConfig.maxLeverage {rebalConfig.maxLeverage} * {sbounds_maxGross}") # constraints.append(cp.max(sGrossExp - maxGross) <= 0.0) pen = cp.sum(cp.pos(sGrossExp - maxGross)) penalty = penalty + pen ## net exposure to sbeta_market mNetExp = fload.reindex(index=assets, columns=['sbeta_market']).to_numpy().T @ w constraints.append(cp.min(mNetExp) >= rebalConfig.minBetaAdjNet) constraints.append(cp.max(mNetExp) <= rebalConfig.maxBetaAdjNet) PyLog.info(f"mNetExp: {mNetExp} rebalConfig.minBetaAdjNet:{rebalConfig.minBetaAdjNet} rebalConfig.maxBetaAdjNet:{rebalConfig.maxBetaAdjNet}") ## phase out value_liq in 2017~ ============================================================ # if PyDate.ge(signalDate, 20170101) and PyDate.le(signalDate, PyMonth.lastWeekday(202012)): # # if PyDate.ge(signalDate, 20170101): # vliqExp = fload.reindex(index=assets, columns=['value_liq']).to_numpy().T @ w # bound = 2.0 * Filter.bound(PyDate.span(signalDate, 20170331) / 90, lower=0.01) # constraints.append(cp.max(vliqExp) <= bound) # constraints.append(cp.min(vliqExp) >= -bound) ## ========================================================================================== ## apply alpha theme exposure bounds ======================================================== if rebalConfig.themeBoundName is not None: themeBounds = SignalMgr.get(rebalConfig.themeBoundName, signalDate, stratName) themeBounds = FrameUtil.toSeries(themeBounds, keyCol='factor', valCol='bound') * rebalConfig.maxLeverage themes = list(themeBounds.index) themeExp = fload.reindex(index=assets, columns=themes).to_numpy().T @ w # constraints.append(cp.max(themeExp - themeBounds.to_numpy().reshape(themeExp.shape)) <= 0) # constraints.append(cp.min(themeExp + themeBounds.to_numpy().reshape(themeExp.shape)) >= 0) pen = cp.sum(cp.pos(themeExp - themeBounds.to_numpy().reshape(themeExp.shape))) \ + cp.sum(cp.pos(- themeExp - themeBounds.to_numpy().reshape(themeExp.shape))) penalty = penalty + pen ## here we are constructing symmetric bounds around 0 but it is the upper bound that we are ## primarily concerned about ## ========================================================================================== ## penalty - apply max trade constraints ==================================================== if (not optimal) and (rebalConfig.maxAdvProp is not None): dfm = SignalMgr.getFrame(rebalConfig.advName, signalDate, stratName) dfm = dframe[['assetKey']].merge(dfm[['assetKey', rebalConfig.advName]], how='left', on='assetKey') dfm = dfm.assign( maxTradeWeights = rebalConfig.maxAdvProp * dfm[rebalConfig.advName].fillna(0.0) * 1000000 / NAV) ## may need to relax this condition for certain conditions (i.e. forced trading) # constraints.append(cp.max(cp.abs(trade) - dfm[['maxTradeWeights']].to_numpy()) <= 0.0) pen = (cp.abs(trade) - dfm[['maxTradeWeights']].to_numpy()) / dfm[['maxTradeWeights']].to_numpy() # pen = 4 * cp.max(cp.pos(pen)) pen = 4 * cp.sum(cp.pos(pen)) penalty = penalty + pen ## ========================================================================================== gamma = cp.Parameter(nonneg=True) if rebalConfig.adaptiveGamma: prevGamma = 0.0 currGamma = 0.0 prevRisk = 0.0 currRisk = 0.0 offset = 1.0 while Real.isZero(currRisk) or \ abs(currRisk - rebalConfig.adaptiveGammaTargetRisk) > rebalConfig.adaptiveGammaTolerance: if Real.isZero(currRisk): rgamma = rebalConfig.gamma elif Real.isZero(prevRisk): rgamma = currGamma + np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) * offset elif Real.isPositive(np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) * np.sign(prevRisk - rebalConfig.adaptiveGammaTargetRisk)): rgamma = currGamma + np.sign(currRisk - rebalConfig.adaptiveGammaTargetRisk) else: rgamma = (prevGamma + currGamma) / 2 offset = offset / 4 gamma.value = gmult * rgamma ## optimization setup if optimal: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - penalty), constraints) else: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - btau * tcost - atau * avgTcost - penalty), constraints) numIters = cls.solveProblem(signalDate, rebalConfig=rebalConfig, problem=problem, maxIter=maxIter, cycleIter=cycleIter, solver=solver, verbose=verbose, checkDCP=checkDCP, optimal=optimal) if w.value is None: PyLog.info(" Optimizer failed to converge") w.value = dframe[['preOptWeights']].fillna(0.0).to_numpy() weights = pd.Series(w.value.flatten(), index=assets) prevGamma = currGamma prevRisk = currRisk currGamma = rgamma currRisk = RiskModel.computeRisk(rmodel, weights) else: rgamma = rebalConfig.gamma gamma.value = gmult * rgamma ## optimization setup if optimal: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - penalty), constraints) else: problem = cp.Problem(cp.Maximize(netReturn - gamma * variance - btau * tcost - atau * avgTcost - penalty), constraints) numIters = cls.solveProblem(signalDate, rebalConfig=rebalConfig, problem=problem, maxIter=maxIter, cycleIter=cycleIter, solver=solver, verbose=verbose, checkDCP=checkDCP, optimal=optimal) if w.value is None: PyLog.info(" Optimizer failed to converge") w.value = dframe[['preOptWeights']].fillna(0.0).to_numpy() weights = pd.Series(w.value.flatten(), index=assets) dfWeights = dfWeights.merge(weights.reset_index().rename(columns={'index': 'assetKey', 0: 'optimalWeights'}), how='outer', on='assetKey') dfWeights = dfWeights.assign(postOptWeights = dfWeights['optimalWeights']) dfWeights.loc[abs(dfWeights['postOptWeights'].fillna(0.0)) < rebalConfig.holdingThreshold, 'postOptWeights'] = 0.0 # weights.loc[abs(weights) < rebalConfig.holdingThreshold] = 0.0 # dfWeights = dfWeights.merge(weights.reset_index().rename(columns={'index': 'assetKey', 0: 'postOptWeights'}), # how='outer', on='assetKey') dctOptResult['numIters'] = numIters ## including cvxpy objects increased output file from 100K to 80MB dctOptResult['gamma'] = rgamma dctOptResult['netReturn'] = netReturn.value dctOptResult['variance'] = variance.value dfWeights = dfWeights.merge(dframe[['assetKey', 'longBoundHard', 'shortBoundHard']], how='left', on='assetKey') dfWeights = dfWeights.assign(longBoundHard = dfWeights['longBoundHard'].fillna(0.0)) dfWeights = dfWeights.assign(shortBoundHard = dfWeights['shortBoundHard'].fillna(0.0)) dfWeights = dfWeights.assign(preOptWeights = dfWeights['preOptWeights'].fillna(0.0)) dfWeights = dfWeights.assign(optimalWeights = dfWeights['optimalWeights'].fillna(0.0)) dfWeights = dfWeights.assign(postOptWeights = dfWeights['postOptWeights'].fillna(0.0)) dfWeights = dfWeights.assign(tradeWeights = dfWeights['postOptWeights'] - dfWeights['preOptWeights']) dctOptResult['dfWeights'] = dfWeights dframe = dfWeights.merge(tmodel.reset_index(), how='left', on='assetKey') linearCost = NAV * ((Filter.bound(dframe['tradeWeights'], lower=0.0) * dframe['linearBuyCoeff'].fillna(dframe['linearBuyCoeff'].max())) - (Filter.bound(dframe['tradeWeights'], upper=0.0) * dframe['linearSellCoeff'].fillna(dframe['linearSellCoeff'].max()))).sum() dctOptResult['linearCostUSD'] = linearCost impactCost = rebalConfig.impactCostScale * np.power(NAV, 3 / 2) * \ (np.power(abs(dframe['tradeWeights']), 3 / 2) * dframe['impactCoeffTH'].fillna(dframe['impactCoeffTH'].max())).sum() dctOptResult['impactCostUSD'] = impactCost dctOptResult['tcostUSD'] = linearCost + impactCost dctOptResult['fcost'] = fcost.value[0] dctOptResult['bcost'] = bcost.value[0] dctOptResult['tradeWeight'] = abs(dfWeights['tradeWeights']).sum() dctOptResult['tradeValUSD'] = dctOptResult['tradeWeight'] * dctOptResult['preOptNAV'] return dctOptResult @classmethod def getExchangeHolidays(cls, tradeDate): hframe = SignalMgr.getStatic('trading_exchange_holidays_master_frame') hframe = hframe[hframe['tradeDate'] == tradeDate] hframe = hframe[hframe['trading'] == 'No'] #lstHolidays = hframe['quoteCountry'].tolist() ### Get unique countries since our holidays table/dataframe has exchange holidays lstHolidays = list(set(hframe['quoteCountry'].tolist())) """ if 'HK' in hframe['quoteCountry']: lstHolidays = list(set(lstHolidays + ['CN', 'XH'])) """ if (hframe['quoteCountry'] == 'HK').any(): lstHolidays = list(set(lstHolidays + ['CN', 'XH'])) PyLog.info(f"Hols={lstHolidays}") return lstHolidays @classmethod def solveProblem(cls, signalDate, rebalConfig, problem, maxIter, cycleIter, solver=cp.ECOS, verbose=True, checkDCP=True, optimal=False): if checkDCP and (not problem.is_dcp()): raise Exception("Optimization failure: problem is not DCP.") numIters = 0 success = True # ============================================================================= # SCS PATH (unchanged, iterative tolerance ladder) # ============================================================================= if solver == cp.SCS: PyLog.info("solveProblemWithClarabel: entering SCS path") targetTolerance = rebalConfig.targetTolerance toleranceRange = rebalConfig.toleranceRange nAttempt = 0 maxAttempt = len(toleranceRange) success = False auxIter = 10 while (not success) and (nAttempt < maxAttempt): PyLog.info(f"SCS attempt {nAttempt} with targetTolerance={targetTolerance:e}") maxCycles = int(np.ceil(maxIter / cycleIter)) nCycles = 0 while (not success) and (nCycles < maxCycles): try: val1 = problem.solve( solver=cp.SCS, max_iters=cycleIter, eps=targetTolerance, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}") except Exception as ee: PyLog.warning(f"SCS cycle failure: caught {ee}") PyLog.warning(traceback.format_exc()) iters = problem.solver_stats.num_iters numIters += iters nCycles += 1 if iters < cycleIter: PyLog.info("SCS converged within cycle") success = True if not success: try: PyLog.info(f"SCS auxiliary solve with tolerance={toleranceRange[nAttempt]:e}") val2 = problem.solve( solver=cp.SCS, max_iters=auxIter, eps=toleranceRange[nAttempt], verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}") except Exception as ee: PyLog.warning(f"SCS auxiliary failure: caught {ee}") PyLog.warning(traceback.format_exc()) iters = problem.solver_stats.num_iters numIters += iters if iters < auxIter: PyLog.info("SCS converged in auxiliary solve") success = True else: PyLog.warning("SCS did not converge, increasing tolerance") nAttempt += 1 return numIters # ============================================================================= # ECOS PATH (unchanged, 2 levels + SCS fallback) # ============================================================================= if solver == cp.ECOS: PyLog.info("solveProblemWithClarabel: entering ECOS path") try: PyLog.info(f"ECOS level one solve {signalDate}") val1 = problem.solve( solver=cp.ECOS, max_iters=maxIter, warm_start=True, abstol=1e-5, reltol=5e-4, feastol=1e-5, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}") except Exception as ee: success = False PyLog.warning(f"ECOS failure level one tolerance. caught: {ee}") PyLog.warning(traceback.format_exc()) if (not success) or (problem.status == "infeasible"): try: PyLog.info("ECOS level two solve (looser tolerances)") success = True val2 = problem.solve( solver=cp.ECOS, max_iters=int(maxIter * 2), warm_start=True, abstol=5e-5, reltol=1e-3, feastol=2e-5, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}") except Exception as ee: success = False PyLog.warning(f"ECOS failure level two tolerance. caught: {ee}") PyLog.warning(traceback.format_exc()) if (not success) or (problem.status == "infeasible"): try: PyLog.warning("ECOS failed or infeasible, falling back to SCS") val3 = problem.solve( solver=cp.SCS, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}") except Exception as ee: PyLog.warning(f"SCS failure third level tolerance. caught: {ee}") PyLog.warning(traceback.format_exc()) return problem.solver_stats.num_iters # ============================================================================= # CLARABEL PATH (new, mirrors ECOS structure) # ============================================================================= if solver == cp.CLARABEL: PyLog.info("solveProblemWithClarabel: entering CLARABEL path") try: PyLog.info(f"CLARABEL level one solve for signal date = {signalDate}") try: val1 = problem.solve( solver=cp.CLARABEL, max_iter=maxIter, warm_start=True, tol_feas=1e-5, tol_gap_abs=1e-5, tol_gap_rel=5e-4, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}") except TypeError: PyLog.info("CLARABEL level one retry without tolerances (compat mode)") val1 = problem.solve( solver=cp.CLARABEL, max_iter=maxIter, warm_start=True, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}") except Exception as ee: success = False PyLog.warning(f"CLARABEL failure level one tolerance. caught: {ee}") PyLog.warning(traceback.format_exc()) if (not success) or (problem.status == "infeasible"): try: PyLog.info(f"CLARABEL level two solve (looser tolerances) for signal date = {signalDate}") success = True try: val2 = problem.solve( solver=cp.CLARABEL, max_iter=int(maxIter * 2), warm_start=True, tol_feas=2e-5, tol_gap_abs=5e-5, tol_gap_rel=1e-3, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}") except TypeError: PyLog.info("CLARABEL level two retry without tolerances (compat mode)") val2 = problem.solve( solver=cp.CLARABEL, max_iter=int(maxIter * 2), warm_start=True, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}for {signalDate}") except Exception as ee: success = False PyLog.warning(f"CLARABEL failure level two tolerance. caught: {ee} for {signalDate}") PyLog.warning(traceback.format_exc()) if (not success) or (problem.status == "infeasible"): try: PyLog.warning("CLARABEL failed or infeasible, falling back to SCS") val3 = problem.solve( solver=cp.SCS, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters} for {signalDate}") except Exception as ee: PyLog.warning(f"SCS failure third level tolerance. caught: {ee}") PyLog.warning(traceback.format_exc()) return problem.solver_stats.num_iters # ============================================================================= # UNKNOWN SOLVER → ECOS → SCS FALLBACK # ============================================================================= PyLog.warning(f"Unknown solver {solver}, falling back to ECOS/SCS") try: PyLog.info(f"Unknown solver fallback: ECOS for signal date = {signalDate}") problem.solve( solver=cp.ECOS, max_iters=maxIter, warm_start=True, abstol=1e-5, reltol=5e-4, feastol=1e-5, verbose=verbose ) PyLog.info(f"solver={solver} status={problem.status} iters={problem.solver_stats.num_iters}") except Exception as ee: PyLog.warning(f"ECOS fallback failed. caught: {ee}") PyLog.warning(traceback.format_exc()) try: PyLog.warning("Final fallback: SCS") problem.solve(solver=cp.SCS, verbose=verbose) except Exception as ee2: PyLog.warning(f"SCS final fallback failed. caught: {ee2}") PyLog.warning(traceback.format_exc()) return problem.solver_stats.num_iters @classmethod def post_optimization_validation( cls, optimizer_stats, tradeDate, rebalConfig, dfWeights, hard_fail=False ): context = "post_opt_validation" # ========================= # File logging setup # ========================= def _trade_date_str(td): try: return td.strftime("%Y%m%d") except Exception: return str(td) td_str = _trade_date_str(tradeDate) logdir = os.environ.get("LOGDIR") or "/data/log" out_dir = os.path.join(logdir, "post_opt_results") os.makedirs(out_dir, exist_ok=True) out_path = os.path.join(out_dir, f"post_opt_{td_str}.txt") had_errors = False f = open(out_path, "w", encoding="utf-8") # placeholder header (rewritten at end) f.write(" " * 200 + "\n") f.flush() def log_info(msg): nonlocal had_errors PyLog.info(msg) f.write(str(msg).rstrip("\n") + "\n") f.flush() if "FAILED" in str(msg): had_errors = True def log_optimizer(msg): PyLog.info(msg) f.write(str(msg).rstrip("\n") + "\n") f.flush() log_optimizer("==================================================") log_optimizer("OPTIMIZER OBJECTIVE BREAKDOWN") log_optimizer("==================================================") for k, v in optimizer_stats.items(): log_optimizer(f"{k:20s} = {v}") log_optimizer("==================================================") log_optimizer("") # ========================= # ORIGINAL FUNCTION LOGIC # ========================= if 'postOptWeights' not in dfWeights.columns or 'preOptWeights' not in dfWeights.columns: PyLog.warning(f"[{context}] Missing preOptWeights/postOptWeights; skipping validation for {tradeDate}.") f.write(f"[{context}] Missing preOptWeights/postOptWeights; skipping validation for {tradeDate}.\n") f.flush() f.seek(0) f.write(f"ALL IS WELL in {td_str} post optimization\n") f.close() return True post_w = dfWeights['postOptWeights'].fillna(0.0) pre_w = dfWeights['preOptWeights'].fillna(0.0) trade_w = post_w - pre_w # ---- (1) GROSS CHECK ---- gross = abs(post_w).sum() max_gross = float(getattr(rebalConfig, "maxLeverage", 4.0)) gross_tol = float(getattr(rebalConfig, "postOptGrossTol", 0.20)) gross_limit = max_gross + gross_tol log_info( f"[{context}] {tradeDate} gross={gross:.4f} " f"limit={gross_limit:.4f} (maxLeverage={max_gross:.2f} tol={gross_tol:.2f})" ) if gross > gross_limit: msg = ( f"[{context}] FAILED gross_exceeds_limit @ {tradeDate}: " f"gross={gross:.4f} > limit={gross_limit:.4f}" ) log_info(msg) if hard_fail: f.seek(0) f.write(f"CRITICAL ERRORS in {td_str} post optimization\n") f.close() raise Exception(msg) f.seek(0) f.write(f"CRITICAL ERRORS in {td_str} post optimization\n") f.close() return False # ---- (2) NET CHECK ---- net = post_w.sum() strategy_type = rebalConfig.strategyLongShortType.upper() if strategy_type == "LONG": min_net = float(rebalConfig.longOnlyMinNetExposure) max_net = float(rebalConfig.longOnlyMaxNetExposure) else: min_net = float(rebalConfig.minNetExposure) max_net = float(rebalConfig.maxNetExposure) log_info( f"[{context}] {tradeDate} strategyLongShortType={strategy_type} " f"net={net:.4f} bounds=[{min_net:.4f},{max_net:.4f}]" ) if net < min_net or net > max_net: msg = ( f"[{context}] FAILED net_out_of_bounds @ {tradeDate}: " f"net={net:.4f} not in [{min_net:.4f},{max_net:.4f}]" ) log_info(msg) if hard_fail: f.seek(0) f.write(f"CRITICAL ERRORS in {td_str} post optimization\n") f.close() raise Exception(msg) f.seek(0) f.write(f"CRITICAL ERRORS in {td_str} post optimization\n") f.close() return False # Skip turnover validation on the initial portfolio build from empty. # This is expected to be ~100% turnover and should not fail post-opt validation. skip_turnover_check = bool( getattr(rebalConfig, "initializeEmptyRebalDateAsStartTD", False) ) and abs(pre_w).sum() < 1e-12 # ---- (3) TURNOVER CHECK ---- turnover = abs(trade_w).sum() if skip_turnover_check: log_info( f"[{context}] {tradeDate} skipping turnover check " f"(initial portfolio build from empty positions)" ) elif gross > 1e-12: turnover_ratio = turnover / gross max_turnover_ratio = float(getattr(rebalConfig, "postOptMaxTurnover", 0.20)) turnover_pct = turnover_ratio * 100.0 max_turnover_pct = max_turnover_ratio * 100.0 log_info( f"[{context}] {tradeDate} turnover={turnover:.4f} " f"turnover_percent={turnover_pct:.2f}% limit={max_turnover_pct:.2f}%" ) if turnover_ratio > max_turnover_ratio: msg = ( f"[{context}] FAILED turnover={turnover:.4f} turnover exceeds limit for {tradeDate}: " f"turnover_percent={turnover_pct:.2f}% > limit={max_turnover_pct:.2f}" ) log_info(msg) if hard_fail: f.seek(0) f.write(f"CRITICAL ERRORS in {td_str} post optimization\n") f.close() raise Exception(msg) f.seek(0) f.write(f"CRITICAL ERRORS in {td_str} post optimization\n") f.close() return False else: log_info(f"[{context}] {tradeDate} gross≈0 and turnover≈0 (turnover check skipped).") # ---- (4) MAX SINGLE NAME ---- max_abs_name = abs(post_w).max() if len(post_w) else 0.0 max_name_abs = getattr(rebalConfig, "postOptMaxAbsName", None) if max_name_abs is not None: log_info( f"[{context}] {tradeDate} max|w|={max_abs_name:.4f} " f"limit={float(max_name_abs):.4f}" ) if max_abs_name > float(max_name_abs): msg = ( f"[{context}] FAILED name_concentration_exceeds_limit @ {tradeDate}: " f"max|w|={max_abs_name:.4f}" ) log_info(msg) if hard_fail: f.seek(0) f.write(f"CRITICAL ERRORS in {td_str} post optimization\n") f.close() raise Exception(msg) f.seek(0) f.write(f"CRITICAL ERRORS in {td_str} post optimization\n") f.close() return False else: log_info(f"[{context}] {tradeDate} max|w|={max_abs_name:.4f} (no limit configured)") log_info(f"[{context}] PASSED all checks @ {tradeDate}.") # ========================= # Final header write # ========================= f.seek(0) if had_errors: f.write(f"CRITICAL ERRORS in {td_str} post optimization\n") else: f.write(f"ALL IS WELL in {td_str} post optimization\n") f.close() return True @classmethod def OLDsolveProblem(cls, rebalConfig, problem, maxIter=200000, cycleIter=5000, solver=cp.ECOS, verbose=False, checkDCP=False, optimal=False): if checkDCP: dcpFlag = problem.is_dcp() PyLog.info('Problem is DCP (Disciplined Convex Problem): {}'.format(dcpFlag)) PyLog.assertion(dcpFlag, "Optimization problem is not DCP (Disciplined Convex Problem)") if solver == cp.SCS: targetTolerance = rebalConfig.targetTolerance toleranceRange = rebalConfig.toleranceRange nAttempt = 0 maxAttempt = len(toleranceRange) success = False auxIter = 10 numIters = 0 while (not success) and (nAttempt < maxAttempt): maxCycles = int(np.ceil(maxIter / cycleIter)) nCycles = 0 cumIters = 0 while (not success) and (nCycles < maxCycles): tol = targetTolerance try: problem.solve(verbose=verbose, solver=solver, max_iters=cycleIter, eps=tol) except: pass cumIters += problem.solver_stats.num_iters numIters += problem.solver_stats.num_iters nCycles += 1 PyLog.info('tau : {:.2f} / numIters : {} / cumIters : {} / tol : {:e}'.format( 0.0 if optimal else rtau, problem.solver_stats.num_iters, cumIters, tol)) if (problem.solver_stats.num_iters < cycleIter): success = True if (problem.solver_stats.num_iters < cycleIter): success = True else: tol = toleranceRange[nAttempt] try: problem.solve(verbose=verbose, solver=solver, max_iters=auxIter, eps=tol) except: pass PyLog.info('tau : {:.2f} / numIters : {} / tol : {:e}'.format( 0.0 if optimal else rtau, problem.solver_stats.num_iters, tol)) numIters += problem.solver_stats.num_iters if (problem.solver_stats.num_iters < auxIter): success = True else: nAttempt += 1 else: try: success = True PyLog.info(f"level one ECOS solve ") #problem.solve(verbose=verbose, solver=cp.ECOS, max_iters=maxIter, abstol=1e-6, reltol=1e-4, feastol=2e-6, abstol_inacc=2e-6, reltol_inacc=2e-4, feastol_inacc=2e-6) val1 = problem.solve(verbose=verbose, solver=cp.ECOS, max_iters=maxIter, warm_start=True, abstol=1e-5, reltol=5e-4, feastol=1e-5) #CLARABEL : cvxpy deprecated ECOS in May 2024 #PyLog.info(f"level one CLARABEL solve") #problem.solve(verbose=verbose, solver=cp.CLARABEL) except Exception as ee: success = False PyLog.warning(f"Error: optimization failure first level tolerance. td: caught: {ee}") PyLog.warning(traceback.format_exc()) if (not success) or (problem.status == 'infeasible'): try: success = True PyLog.info(f"level two ECOS solve ") #problem.solve(verbose=verbose, solver=cp.ECOS, max_iters=maxIter, # abstol=1e-6, reltol=5e-4, feastol=2e-6, # abstol_inacc=2e-6, reltol_inacc=1e-3, feastol_inacc=2e-6) # --- Level 2 (fallback, looser than Level 1) --- val2 = problem.solve( solver=cp.ECOS, max_iters=int(maxIter * 2), # give the fallback extra room warm_start=True, abstol=5e-5, reltol=1e-3, feastol=2e-5, verbose=verbose ) except Exception as ee: success = False PyLog.warning(f"Error: ECOS failure second level tolerance. td: caught: {ee}") PyLog.warning(traceback.format_exc()) if (not success) or (problem.status == 'infeasible'): try: PyLog.info(f"level three SCS solve ") #problem.solve(verbose=verbose, solver=cp.ECOS, max_iters=maxIter, # abstol=1e-6, reltol=5e-3, feastol=2e-6, # abstol_inacc=2e-6, reltol_inacc=1e-2, feastol_inacc=2e-6) problem.solve(verbose=verbose, solver=cp.SCS) except Exception as ee: PyLog.warning(f"SCS failure third level tolerance. td: caught: {ee}") PyLog.warning(traceback.format_exc()) numIters = problem.solver_stats.num_iters return numIters @classmethod def runOptimal(cls, rebalConfig=RebalConfig_DEFAULT, signalDate=PyDate.asDate(20191115), NAV=100_000_000, solver=cp.SCS, maxIter=200000, cycleIter=5000, verbose=True, checkDCP=True, tradeRestrictions=None): return cls.run(rebalConfig=rebalConfig, signalDate=signalDate, preOptWeights=cls.EMPTY_PORTFOLIO, NAV=NAV, solver=solver, maxIter=maxIter, cycleIter=cycleIter, verbose=verbose, checkDCP=checkDCP, optimal=True, tradeRestrictions=tradeRestrictions) @classmethod def generateRiskReport(cls, dctOptResult, weightCol='postOptWeights', maxReturnDate=PyDate.today()): rebalConfig = dctOptResult['rebalConfig'] portWeights = FrameUtil.toSeries(dctOptResult['dfWeights'], keyCol='assetKey', valCol=weightCol) return RiskReport.runPortfolio(port = portWeights, signalDate = dctOptResult['signalDate'], alphaName = rebalConfig.alphaName, fcostName = rebalConfig.fcostName, bcostName = rebalConfig.bcostName, rmodelName = rebalConfig.rmodelName, envName = Strategy.getModelName(rebalConfig.stratName), minWeight = rebalConfig.holdingThreshold, maxReturnDate = maxReturnDate) @classmethod def printRiskReport(cls, dctOptResult, plotChart=False): dctReport = cls.generateRiskReport(dctOptResult) return RiskReport.print(dctReport, plotChart) @classmethod def compileShortAvailability(cls, rebalConfig, NAV, tradeDate, dframe): lookback = rebalConfig.bcostLookback endTD = tradeDate startTD = PyDate.minusWeekdays(endTD, lookback - 1) dfWgts = pd.DataFrame({'tradeDate': PyDate.sequenceWeekday(startTD, endTD, decreasing=True), 'timeWgts': Stats.expwts(lookback, lookback/2)}) dfWgts = dfWgts.assign(timeScale = dfWgts['timeWgts'] / dfWgts['timeWgts'].iloc[0]) dfRaw = ShortAvailability.getRange(startTD, endTD, pbList=rebalConfig.pbList) dfRaw = dfRaw[dfRaw['assetKey'].isin(dframe['assetKey'])] ################### CHANGED BY RD HACK FOR INDIA ############### if len(dfRaw) == 0: PyLog.info(f"dfRaw is zero No short availability found. " "Using dframe as dfAvail.") dfAvail = dframe.copy() dfAvail["bcost"] = 0.0 dfAvail["tier"] = 0 dfAvail["key"] = (dfAvail["assetKey"] + ":0") dfMarginal = dframe[["assetKey"] ].copy() dfMarginal["availablePct"] = 1.0 return dfAvail, dfMarginal ################### CHANGED BY RD HACK FOR INDIA ############### if rebalConfig.HAUM: AUM = rebalConfig.hypotheticalAUM else: AUM = NAV ########## THIS BLOCK IS NOT NEEDED SINCE WE DON'T HAVE JPM AS BROKER ############ """ dfRaw = dfRaw.assign( scale=rebalConfig.maxShortUtilization + (1 - rebalConfig.maxShortUtilization) * ((dfRaw['pbCode'] == 'JPM') & (dfRaw['category'] == 'connect')).astype(int)) """ ########## THIS BLOCK IS NOT NEEDED SINCE WE DON'T HAVE JPM AS BROKER ############ # So max shorting depends only only this config variable. dfRaw = dfRaw.assign(scale = rebalConfig.maxShortUtilization) dfRaw = dfRaw.assign(vsHAUM=dfRaw['scale'] * dfRaw['notionalUSD'] / AUM) dfRaw = dfRaw.merge(dfWgts[['tradeDate', 'timeScale']], how='left', on='tradeDate') ## availableRate is float and it works poorly with group() dfRaw = dfRaw.assign(rateClass=['{:.4f}'.format(x) for x in dfRaw['availableRate']]) dfMarginal = dfRaw[dfRaw['tradeDate'] == tradeDate].\ groupby('assetKey').agg({'vsHAUM': sum}).reset_index().\ rename(columns={'vsHAUM': 'availablePct'}) ### THIS IS INCORRECTLY SUMMING UP ALL THE EXPONENTIAL DAYS ### #dfConsol = dfRaw.groupby(['assetKey', 'rateClass']). \ #apply(lambda dfm: (dfm['vsHAUM'] * dfm['timeScale']).sum()). \ #reset_index().rename(columns={0: 'availablePct'}) dfConsol = dfRaw.groupby(['assetKey', 'rateClass']) \ .apply(lambda dfm: (dfm['vsHAUM'] * dfm['timeScale']).sum() / dfm['timeScale'].sum()) \ .reset_index().rename(columns={0: 'availablePct'}) dfConsol = dfConsol.assign(availableRate=dfConsol['rateClass'].astype(float)) dfConsol = dfConsol.sort_values(by=['assetKey', 'availableRate'], ascending=True). \ groupby('assetKey').apply(lambda dfm: dfm.assign(cumHi=dfm['availablePct'].cumsum())).reset_index(drop=True) dfConsol = dfConsol.groupby('assetKey'). \ apply(lambda dfm: dfm.assign(cumLo=[0.0] + dfm['cumHi'][:-1].tolist())).reset_index(drop=True) dfConsol = dfConsol.merge(dframe[['assetKey', 'preOptWeights']], how='left', on='assetKey') ## discard irrelevant availability softShortBoundBuffer = 0.01 dfConsol = dfConsol.merge(dframe[['assetKey', 'shortBoundSoft', 'shortBoundHard']], how='left', on='assetKey') dfConsol = dfConsol.assign(bufferedShortBoundSoft=dfConsol['shortBoundSoft'] + softShortBoundBuffer) dfConsol = dfConsol[dfConsol['cumLo'] <= dfConsol['shortBoundHard']] dfConsol = dfConsol[dfConsol['cumLo'] <= dfConsol['bufferedShortBoundSoft']] ## technically, this is not strict but we are going to simplify the logic and keep track of ## all availability and marginal availability dfConsol = dfConsol.sort_values(by=['assetKey', 'availableRate'], ascending=True). \ groupby('assetKey').apply(lambda dfm: dfm.assign(tier=range(len(dfm)))).reset_index(drop=True) columns = ['assetKey', 'tier', 'availableRate', 'availablePct'] dfBaseTier = dfConsol[dfConsol['tier'] == 0][columns].merge(dframe, how='inner', on='assetKey') #dfBaseTier = dfBaseTier.assign(bcost=dfBaseTier['availableRate'] / 100) dfBaseTier = dfBaseTier.assign(bcost=dfBaseTier['availableRate'] ) # dfBaseTier = dfBaseTier.assign(shortBoundHard = dfBaseTier[['shortBoundHard', 'availablePct']].min(axis=1)) dfBaseTier = dfBaseTier.assign(shortBoundHard=dfBaseTier['availablePct']) dfBaseTier = dfBaseTier.drop(columns=['availableRate', 'availablePct']) dfHighTier = dfConsol[dfConsol['tier'] != 0][['assetKey', 'tier', 'availableRate', 'availablePct']]. \ merge(dframe, how='inner', on='assetKey') dfHighTier = dfHighTier.assign(longBoundHard=0.0) dfHighTier = dfHighTier.assign(shortBoundHard=dfHighTier['availablePct']) #dfHighTier = dfHighTier.assign(bcost=dfHighTier['availableRate'] / 100) dfHighTier = dfHighTier.assign(bcost=dfHighTier['availableRate'] ) dfHighTier = dfHighTier.assign(fcost=1.00) dfHighTier = dfHighTier.assign(preOptWeights=np.NaN) dfHighTier = dfHighTier.drop(columns=['availableRate', 'availablePct']) ## no short availability dfNoAvail = dframe[~dframe['assetKey'].isin(dfConsol['assetKey'])] dfNoAvail = dfNoAvail.assign(shortBoundHard=0.0) dfNoAvail = dfNoAvail.assign(bcost=1.0) dfNoAvail = dfNoAvail.assign(tier=-1) dfAvail = pd.concat([dfBaseTier, dfHighTier, dfNoAvail], ignore_index=True) dfAvail = dfAvail.assign(key=dfAvail['assetKey'] + ':' + dfAvail['tier'].astype(str)) dfAvail = dfAvail.sort_values(by=['assetKey', 'tier']) ## bcostMultiplier if rebalConfig.bcostMultiplierName is not None: signalDate = PyDate.prevWeekday(tradeDate) signalObj = Signal.registryLookup(rebalConfig.bcostMultiplierName) dfm = SignalMgr.getFrame(signalObj.signalName, signalDate).\ rename(columns={signalObj.signalName: 'multiplier'}) dfAvail = dfAvail.merge(dfm[['assetKey', 'multiplier']], how='left', on='assetKey') dfAvail = dfAvail.assign(multiplier = dfAvail['multiplier'].fillna(signalObj.maxMultiplier)) dfAvail = dfAvail.assign(bcost = dfAvail['bcost'] * dfAvail['multiplier']) dfAvail = dfAvail.drop(columns='multiplier') return dfAvail, dfMarginal def load_china_connect_inventory_scores(input_date, max_lookback_days): """ Try to read China Connect Inventory Score file for the given signal date. From what I have noticed, for tradeDate say 11/19, we get files for 11/18 and that too at 9pm NY time but our optimizer runs at 3pm NY time so these 11/18 files come late. So we process only two days earlier files for 11/17. Hence I have put in logic for max_lookback_days as 3 so we will try to get trade_date(say 11/19) files, if not (11/18), if not (11/17) Returns dataframe ['assetKey','stabilityScore'] or None. """ import PyUtil.PyRicX as RicX try: pbdata_dir = os.environ.get('PBDIR') if not pbdata_dir: PyLog.info("ChinaConnect: PBDIR env var not set; skipping China Connect short penalty.") return None base_dir = os.path.join(pbdata_dir, "GS_Stability_Files") if not os.path.isdir(base_dir): PyLog.info(f"ChinaConnect: directory {base_dir} does not exist; skipping China Connect short penalty.") return None # ---------------------------------------------------------- # NEW: Loop over working days (signalDate, -1 WD, -2 WD, etc.) # ---------------------------------------------------------- dates_to_try = [] d = PyDate.asDate(input_date) for _ in range(max_lookback_days): dates_to_try.append(d) d = PyDate.prevWeekday(d) filename = None for d in dates_to_try: date_str = d.strftime("%Y%m%d") pattern_xls = os.path.join(base_dir, f"*China_Connect*In*{date_str}.xls") pattern_xlsx = os.path.join(base_dir, f"*China_Connect*In*{date_str}.xlsx") files = glob.glob(pattern_xls) + glob.glob(pattern_xlsx) if files: files.sort(key=os.path.getmtime) filename = files[-1] # newest file for that date PyLog.info(f"ChinaConnect: using inventory score file {filename} (date {date_str})") break else: PyLog.info(f"ChinaConnect: no file for date {date_str}, trying previous working day...") if not filename: PyLog.info(f"ChinaConnect: no China_Connect file found in last {max_lookback_days} working days; skipping penalty.") return None # ---------------------------------------------------------- # Read without header; find the row where the table starts raw = pd.read_excel(filename, header=None) col0 = raw.iloc[:, 0].astype(str).str.strip() header_rows = raw.index[col0 == "RIC"].tolist() if not header_rows: PyLog.info(f"ChinaConnect: could not find header row with 'RIC' in {filename}; skipping penalty.") return None h = header_rows[0] df = raw.iloc[h:].copy() df.columns = df.iloc[0] df = df.iloc[1:] # drop header row df = df.rename(columns={c: str(c).strip() for c in df.columns}) expected_cols = {"RIC", "SCORE", "SEDOL", "ISIN"} if not expected_cols.issubset(df.columns): PyLog.info(f"ChinaConnect: expected columns {expected_cols} not found; skipping penalty.") return None df = df[["RIC", "SCORE", "SEDOL", "ISIN"]].copy() df["RIC"] = df["RIC"].astype(str).str.strip() df["SCORE"] = pd.to_numeric(df["SCORE"], errors="coerce") # Keep only ZK and SH mask = df["RIC"].str.endswith(".ZK") | df["RIC"].str.endswith(".SH") df = df[mask].copy() if df.empty: PyLog.info("ChinaConnect: no .ZK/.SH rows in file; nothing to do.") return None # Convert to internal_ric: ZK->SZ, SH->SS df["internal_ric"] = df["RIC"] df.loc[df["RIC"].str.endswith(".ZK"), "internal_ric"] = df["RIC"].str.replace(".ZK", ".SZ", regex=False) df.loc[df["RIC"].str.endswith(".SH"), "internal_ric"] = df["RIC"].str.replace(".SH", ".SS", regex=False) # Map to assetKey df = RicX.add_asset_code(df, "internal_ric") if "assetKey" not in df.columns: PyLog.info("ChinaConnect: add_asset_code failed; skipping penalty.") return None df = df[["assetKey", "SCORE"]].dropna() if df.empty: PyLog.info("ChinaConnect: no usable rows remain; skipping penalty.") return None # Aggregate duplicates by max score df = df.groupby("assetKey", as_index=False)["SCORE"].max() df = df.rename(columns={"SCORE": "stabilityScore"}) PyLog.info(f"ChinaConnect: loaded {len(df)} inventory score rows with assetKey.") return df except Exception as e: PyLog.error(f"ChinaConnect: exception while loading inventory scores: {e}") PyLog.error(traceback.format_exc()) return None def build_china_connect_short_penalty(dfAvail, wshort, input_date): # For now we will hardcode max_lookback_days here, ideally it should be in rebalance.config max_lookback_days = 3 # Load scores from file df_scores = load_china_connect_inventory_scores(input_date, max_lookback_days) if df_scores is None or df_scores.empty: return None # Map assetKey -> stabilityScore df_scores = df_scores.set_index("assetKey")["stabilityScore"] if "assetKey" not in dfAvail.columns: PyLog.info("ChinaConnect: dfAvail has no 'assetKey' column; cannot apply stability penalty.") return None n = len(dfAvail) penalties = np.zeros(n) penalized_assets = [] # (idx, assetKey, score, penaltyWeight) # Build penalty weights per dfAvail row for idx, a in enumerate(dfAvail["assetKey"]): if a in df_scores.index: s = df_scores.loc[a] if pd.notna(s) and s > 4: # 5→1, 6→2, 7→3, 8→4 (can be tuned) pw = max(0.0, float(s) - 4.0) penalties[idx] = pw penalized_assets.append((idx, a, float(s), pw)) if not penalized_assets: PyLog.info("ChinaConnect: no dfAvail assets with stabilityScore > 4; no penalty applied.") return None # Log penalized assets BEFORE solve (no short weights yet) PyLog.info(f"ChinaConnect: {len(penalized_assets)} dfAvail assets will have short-side stability penalty.") PyLog.info("ChinaConnect: penalized assets (before solve, top 50):") for idx, a, s, pw in penalized_assets[:50]: PyLog.info(f" idx={idx}, assetKey={a}, score={s}, penaltyWeight={pw}") if len(penalized_assets) > 50: PyLog.info(f" ...and {len(penalized_assets) - 50} more penalized assets") # Optional tuning knob (strength of this penalty) china_scale = 0.1 # adjust as needed or make configurable # penalties is constant data; cvxpy treats it as coefficients penalties_vec = penalties.reshape(-1, 1) penalty_expr = china_scale * cp.sum(cp.multiply(penalties_vec, wshort)) # Attach metadata so we can inspect after solve penalty_expr._china_penalty_info = penalized_assets penalty_expr._china_penalty_weights = penalties_vec penalty_expr._china_scale = china_scale PyLog.info("ChinaConnect: short-side stability penalty term added to objective.") return penalty_expr def build_trbc_cluster_penalty( dframe, assets, aggrw, tradeDate, gqr_dir=GQR_SPECRET_DIR, corr_threshold=0.5, min_cluster_size=2, lambda_cluster=0.5, penalize_by="PENALIZE_NET_EXPOSURE", ): activity_col = "Activity" # If we don't have activity information, bail out quietly if activity_col not in dframe.columns: PyLog.info(f"TRBC cluster penalty: column {activity_col} not found → skipping") return 0 date_str = tradeDate.strftime("%Y%m%d") var_path = os.path.join(gqr_dir, f"gqr_specific_var.{date_str}.csv") scov_path = os.path.join(gqr_dir, f"gqr_scov_sparse.{date_str}.csv") if not os.path.exists(var_path) or not os.path.exists(scov_path): PyLog.info( f"TRBC cluster penalty: missing var/scov for {date_str} " f"(var={os.path.exists(var_path)}, scov={os.path.exists(scov_path)}) → skipping" ) return 0, None try: var_df = pd.read_csv(var_path) var_df = var_df.set_index("assetKey")["spec_var"].to_dict() scov_df = pd.read_csv(scov_path) except Exception as e: PyLog.info(f"TRBC cluster penalty: failed to read var/scov for {date_str}: {e} → skipping") return 0, None # Map assetKey -> activity for current universe df_act = ( dframe[["assetKey", activity_col]] .dropna() .drop_duplicates(subset=["assetKey"]) .set_index("assetKey") ) act_map = df_act[activity_col].to_dict() # Map assetKey -> index in aggrw key_to_idx = {} for i, k in enumerate(assets): key_to_idx[k] = i # Build adjacency by activity using high-|corr| pairs adjacency = {} edges_used = 0 for _, row in scov_df.iterrows(): x = row["xKey"] y = row["yKey"] ax = act_map.get(x, None) ay = act_map.get(y, None) if ax is None or ay is None: continue # Only within same TRBC activity if ax != ay: continue vx = var_df.get(x, None) vy = var_df.get(y, None) if vx is None or vy is None or vx <= 0 or vy <= 0: continue cov_xy = row["cov"] # Convert sparse specific covariance to correlation corr_xy = cov_xy / math.sqrt(vx * vy) if abs(corr_xy) < corr_threshold: continue # Only keep edges where both sides are in the optimization universe if x not in key_to_idx or y not in key_to_idx: continue i = key_to_idx[x] j = key_to_idx[y] if ax not in adjacency: adjacency[ax] = {} if i not in adjacency[ax]: adjacency[ax][i] = set() if j not in adjacency[ax]: adjacency[ax][j] = set() adjacency[ax][i].add(j) adjacency[ax][j].add(i) edges_used += 1 if edges_used == 0: PyLog.info(f"TRBC cluster penalty: no within-activity high-corr pairs for {date_str} → skipping") return 0, None # Find connected components (clusters) within each activity clusters = [] for act, adj in adjacency.items(): visited = set() for node in list(adj.keys()): if node in visited: continue # BFS / DFS to collect a cluster stack = [node] comp = [] while stack: u = stack.pop() if u in visited: continue visited.add(u) comp.append(u) for v in adj.get(u, []): if v not in visited: stack.append(v) if len(comp) >= min_cluster_size: clusters.append((act, sorted(comp))) if not clusters: PyLog.info( f"TRBC cluster penalty: no clusters of size >= {min_cluster_size} for {date_str} → skipping" ) return 0, None # Write clusters out so we can log exposures later try: rows = [] for cluster_id, (act, idx_list) in enumerate(clusters): for idx in idx_list: rows.append( { "date": date_str, "activity": act, "cluster_id": act, "idx": idx, "assetKey": assets[idx], } ) cluster_df = pd.DataFrame(rows) except Exception as e: PyLog.info(f"TRBC cluster penalty: failed to write clusters CSV for {date_str}: {e}") # --- NEW: build penalty expression depending on penalize_by mode --- penalty_terms = [] # Normalise flag and accept both the "correct" and typo'd third-mode name flag = (penalize_by or "PENALIZE_NET_EXPOSURE").upper() high_pe_high_vol_flags = { "PENALIZE_NET_EXPOSURE_HIGH_PE_AND_HIGH_VOLATILE_SECTORS", } is_high_pe_high_vol_mode = flag in high_pe_high_vol_flags # In the special third mode, pre-compute which Activities are both high-vol and high-P/E penalized_activities = None if is_high_pe_high_vol_mode: # Activity-level volatility from per-stock 'annualized' act_vol = compute_activity_volatility( dframe, activity_col=activity_col, vol_col="annualized", agg_method="median", ) # Activity-level P/E (currently returns {} until I wire pe_col) act_pe = compute_activity_pe( dframe, activity_col=activity_col, pe_col=None, agg_method="median", ) if not act_vol or not act_pe: PyLog.info( "TRBC cluster penalty: high-PE/high-vol mode selected but missing " "volatility or P/E data → no Activities will be penalized in this mode." ) penalized_activities = set() # effectively disables this mode else: # thresholds USE_RELATIVE_THRESHOLDS = False # set True to switch to relative mode VOL_ABS_THRESHOLD = 0.40 # 40% annualised vol PE_ABS_THRESHOLD = 20.0 # P/E > 20 RELATIVE_TOP_FRACTION = 0.30 # top 30% by vol and P/E vol_values = np.array(list(act_vol.values()), dtype=float) pe_values = np.array(list(act_pe.values()), dtype=float) if USE_RELATIVE_THRESHOLDS: # Top 30% = 70th percentile cutoff vol_cutoff = float(np.quantile(vol_values, 1.0 - RELATIVE_TOP_FRACTION)) pe_cutoff = float(np.quantile(pe_values, 1.0 - RELATIVE_TOP_FRACTION)) else: vol_cutoff = VOL_ABS_THRESHOLD pe_cutoff = PE_ABS_THRESHOLD high_vol_acts = { a for a, v in act_vol.items() if v is not None and v >= vol_cutoff } high_pe_acts = { a for a, v in act_pe.items() if v is not None and v >= pe_cutoff } penalized_activities = high_vol_acts & high_pe_acts PyLog.info( f"TRBC cluster penalty: high-PE/high-vol mode → " f"{len(penalized_activities)} activities meet both criteria " f"(vol_cutoff={vol_cutoff:.3f}, pe_cutoff={pe_cutoff:.3f})" ) # Build penalty per cluster for act, idx_list in clusters: if flag == "PENALIZE_ALL_STOCKS": # Penalise each stock in the cluster: sum_i w_i^2 cluster_pen = cp.sum_squares(aggrw[idx_list]) penalty_terms.append(cluster_pen) elif flag == "PENALIZE_NET_EXPOSURE": # Penalise only net exposure: (sum_i w_i)^2 exposure_C = cp.sum(aggrw[idx_list]) penalty_terms.append(cp.square(exposure_C)) elif is_high_pe_high_vol_mode: # Only penalize clusters whose Activity is both high-vol and high-P/E if penalized_activities and act not in penalized_activities: continue exposure_C = cp.sum(aggrw[idx_list]) penalty_terms.append(cp.square(exposure_C)) else: # Fallback: default to net-exposure penalty if unknown flag exposure_C = cp.sum(aggrw[idx_list]) penalty_terms.append(cp.square(exposure_C)) if not penalty_terms: PyLog.info( "TRBC cluster penalty: no clusters selected under penalize_by=" f"{penalize_by} → returning 0" ) return 0 pen_expr = lambda_cluster * cp.sum(penalty_terms) PyLog.info( f"TRBC cluster penalty: built {len(clusters)} clusters from {edges_used} edges for {date_str}, " f"lambda={lambda_cluster}, penalize_by={penalize_by}" ) return pen_expr, cluster_df def apply_high_vol_activity_hard_bounds( dframe, dfAvail=None, activity_col="Activity", vol_col="annualized", vol_abs_threshold=0.40, tol=1e-8, tradeDate=None, ): # 0) Ensure we have volatility in dframe; if not, pull Parkinson vol here if vol_col not in dframe.columns: try: df_parkinson = SignalMgr.get("parkinson_volatility_frame", tradeDate) except Exception as e: PyLog.info(f"apply_high_vol_activity_hard_bounds: failed SignalMgr.get parkinson_volatility_frame: {e}") return dframe, dfAvail, False if df_parkinson is None or len(df_parkinson) == 0: PyLog.info("apply_high_vol_activity_hard_bounds: parkinson_volatility_frame empty → skipping") return dframe, dfAvail, False needed = [c for c in ["assetKey", "annualized"] if c in df_parkinson.columns] if "assetKey" not in needed or "annualized" not in needed: PyLog.info("apply_high_vol_activity_hard_bounds: parkinson frame missing assetKey/annualized → skipping") return dframe, dfAvail, False df_parkinson = df_parkinson[needed].drop_duplicates(subset=["assetKey"]) dframe = dframe.merge(df_parkinson, how="left", on="assetKey") PyLog.info( f"apply_high_vol_activity_hard_bounds: merged annualized vol for " f"{dframe['annualized'].notna().sum()} assets" ) # Activity-level volatility act_vol = compute_activity_volatility( dframe, activity_col=activity_col, vol_col=vol_col, agg_method="median", ) if not act_vol: PyLog.info("apply_high_vol_activity_hard_bounds: no activity vol → skipping") return dframe, dfAvail, False # Pick Activities that are "high-vol" bad_activities = { act for act, v in act_vol.items() if v is not None and v >= vol_abs_threshold } if not bad_activities: PyLog.info(f"apply_high_vol_activity_hard_bounds: no activities above {vol_abs_threshold:.3f}") # Construct EMPTY DataFrame with correct columns out_df = pd.DataFrame(columns=[ "assetKey", activity_col, "preOptWeights", "longBoundHard", "shortBoundHard", "tradeDate" ]) # Fill tradeDate so filename is correct date_str = tradeDate.strftime("%Y%m%d") if tradeDate is not None else "UNKNOWN" logdir = os.environ.get("LOGDIR", "/data/log") os.makedirs(os.path.join(logdir, "trbc_cluster_penalty"), exist_ok=True) excel_path = os.path.join(logdir, "trbc_cluster_penalty", f"trbc_cluster_report.{date_str}.xlsx") # Write EMPTY sheet with headers only with pd.ExcelWriter(excel_path, engine="openpyxl") as writer: out_df.to_excel(writer, sheet_name="HighVol_Activity_Bounds", index=False) PyLog.info(f"apply_high_vol_activity_hard_bounds: created EMPTY Excel report {excel_path}") return dframe, dfAvail, False # Sanity checks for required columns if activity_col not in dframe.columns: PyLog.info("apply_high_vol_activity_hard_bounds: activity column missing → skipping") return dframe, dfAvail, False if "preOptWeights" not in dframe.columns: PyLog.info("apply_high_vol_activity_hard_bounds: preOptWeights column missing → skipping") return dframe, dfAvail, False if "longBoundHard" not in dframe.columns or "shortBoundHard" not in dframe.columns: PyLog.info("apply_high_vol_activity_hard_bounds: hard bound columns missing → skipping") return dframe, dfAvail, False # Identify rows in high-vol Activities mask_bad = dframe[activity_col].isin(bad_activities).to_numpy() if not mask_bad.any(): PyLog.info("apply_high_vol_activity_hard_bounds: no rows in bad activities → skipping") return dframe, dfAvail, False # Get current weights and bounds as numpy arrays cur_w = dframe["preOptWeights"].astype(float).fillna(0.0).to_numpy() lbh = dframe["longBoundHard"].astype(float).fillna(0.0).to_numpy() sbh = dframe["shortBoundHard"].astype(float).fillna(0.0).to_numpy() # For each name in a high-vol Activity: # If currently long: lbh = min(lbh, current_long), sbh = 0 # If currently short: sbh = min(sbh, |current_short|), lbh = 0 # If flat: lbh = 0, sbh = 0 for i in range(len(dframe)): if not mask_bad[i]: continue w0 = cur_w[i] if w0 > tol: lbh[i] = min(lbh[i], w0) sbh[i] = 0.0 elif w0 < -tol: sbh[i] = min(sbh[i], -w0) lbh[i] = 0.0 else: lbh[i] = 0.0 sbh[i] = 0.0 dframe = dframe.copy() dframe["longBoundHard"] = lbh dframe["shortBoundHard"] = sbh n_rows = int(mask_bad.sum()) PyLog.info( f"apply_high_vol_activity_hard_bounds: {len(bad_activities)} high-vol activities, " f"{n_rows} rows with tightened bounds" ) # Propagate matching bounds into dfAvail (splitLongShort case) if dfAvail is not None and isinstance(dfAvail, pd.DataFrame) and "assetKey" in dfAvail.columns: try: key_to_lbh = dict(zip(dframe["assetKey"], dframe["longBoundHard"])) key_to_sbh = dict(zip(dframe["assetKey"], dframe["shortBoundHard"])) bad_keys = set(dframe.loc[mask_bad, "assetKey"]) mask_av = dfAvail["assetKey"].isin(bad_keys) if "longBoundHard" in dfAvail.columns: dfAvail.loc[mask_av, "longBoundHard"] = dfAvail.loc[mask_av, "assetKey"].map(key_to_lbh) if "shortBoundHard" in dfAvail.columns: dfAvail.loc[mask_av, "shortBoundHard"] = dfAvail.loc[mask_av, "assetKey"].map(key_to_sbh) except Exception as e: PyLog.info(f"apply_high_vol_activity_hard_bounds: failed to update dfAvail: {e}") ### Create high-vol activity bounds CSV + Excel Tab 1 try: out_df = dframe.loc[ mask_bad, ["assetKey", activity_col, "preOptWeights", "longBoundHard", "shortBoundHard"] ].copy() out_df["tradeDate"] = tradeDate.strftime("%Y%m%d") if tradeDate is not None else None logdir = os.environ.get("LOGDIR", "/data/log") os.makedirs(os.path.join(logdir, "trbc_cluster_penalty"), exist_ok=True) date_str = out_df["tradeDate"].iloc[0] # Now create the Excel report (Tab 1 only) excel_path = os.path.join(logdir, "trbc_cluster_penalty", f"trbc_cluster_report.{date_str}.xlsx") with pd.ExcelWriter(excel_path, engine="openpyxl") as writer: out_df.to_excel(writer, sheet_name="HighVol_Activity_Bounds", index=False) PyLog.info(f"apply_high_vol_activity_hard_bounds: created Excel report {excel_path}") except Exception as e: PyLog.info(f"apply_high_vol_activity_hard_bounds: failed to write Excel/logs: {e}") return dframe, dfAvail, True def compute_activity_volatility( dframe, activity_col="Activity", vol_col="annualized", agg_method="median", ): if activity_col not in dframe.columns or vol_col not in dframe.columns: PyLog.info( f"compute_activity_volatility: missing columns " f"{activity_col} or {vol_col} → returning empty dict" ) return {} df = dframe[[activity_col, vol_col]].dropna() if df.empty: PyLog.info("compute_activity_volatility: no data after dropna → returning empty dict") return {} if agg_method == "median": series = df.groupby(activity_col)[vol_col].median() elif agg_method == "mean": series = df.groupby(activity_col)[vol_col].mean() else: PyLog.info(f"compute_activity_volatility: unknown agg_method={agg_method} → using median") series = df.groupby(activity_col)[vol_col].median() return series.to_dict() def log_trbc_cluster_exposures( tradeDate, assets, aggrw, cluster_df, gqr_dir=GQR_SPECRET_DIR, lambda_cluster=None, ): if cluster_df is None or cluster_df.empty: PyLog.info("TRBC cluster exposure: no cluster_df → skipping") return df_cl = cluster_df.copy() try: date_str = tradeDate.strftime("%Y%m%d") logdir = os.environ.get("LOGDIR", "/data/log") os.makedirs(os.path.join(logdir, "trbc_cluster_penalty"), exist_ok=True) excel_path = os.path.join(logdir, "trbc_cluster_penalty", f"trbc_cluster_report.{date_str}.xlsx") # === Map weights === wvec = np.asarray(aggrw.value).reshape(-1) key_to_idx = {k: i for i, k in enumerate(assets)} df_cl["weight"] = df_cl["assetKey"].map(lambda ak: wvec[key_to_idx[ak]] if ak in key_to_idx else np.nan) df_cl["abs_weight"] = df_cl["weight"].abs() # === Aggregate exposures at cluster level === grp = df_cl.groupby(["date", "activity", "cluster_id"], as_index=False).agg( n_assets=("assetKey", "nunique"), cluster_net_w=("weight", "sum"), cluster_gross_w=("abs_weight", "sum"), max_gross_w=("abs_weight", "max"), ) # === penalty column === if lambda_cluster is not None: grp["cluster_penalty"] = lambda_cluster * grp["cluster_net_w"] ** 2 else: grp["cluster_penalty"] = 0.0 # === Identify max clusters === max_net_row = grp.iloc[grp["cluster_net_w"].abs().idxmax()].to_frame().T max_gross_row = grp.iloc[grp["cluster_gross_w"].idxmax()].to_frame().T max_pen_row = grp.iloc[grp["cluster_penalty"].idxmax()].to_frame().T # === Write Tabs 2–6 to Excel (append mode) === with pd.ExcelWriter(excel_path, mode="a", engine="openpyxl", if_sheet_exists="replace") as writer: df_cl.to_excel(writer, sheet_name="TRBCclusters", index=False) grp.to_excel(writer, sheet_name="TRBCclusterExposure", index=False) max_net_row.to_excel(writer, sheet_name="MaxNetCluster", index=False) max_gross_row.to_excel(writer, sheet_name="MaxGrossCluster", index=False) max_pen_row.to_excel(writer, sheet_name="MaxPenaltyCluster", index=False) PyLog.info(f"TRBC cluster exposure Excel updated → {excel_path}") except Exception as e: PyLog.info(f"TRBC exposure: failed for {tradeDate}: {e}") ######## End of TRBC Activity cluster enhancement #################### # === Conditioning Functions (updated with ModelCntry merge) === def get_returns_history(signalDate, lookback): import datetime as _dt # Handle signalDate as int/str YYYYMMDD or date/datetime try: # if already like 20251128 or "20251128" end_date = _dt.datetime.strptime(str(signalDate), "%Y%m%d") except Exception: # assume it's a date/datetime if isinstance(signalDate, _dt.datetime): end_date = signalDate else: end_date = _dt.datetime.combine(signalDate, _dt.time.min) start_date = end_date - _dt.timedelta(days=lookback) frames = [] print(f"ALPHA CONIDTIONING START DATE {start_date} END DATE{end_date}") for cur in pd.date_range(start_date, end_date): # skip weekends if cur.weekday() in (5, 6): continue cur_str = cur.strftime("%Y%m%d") rf = SignalMgr.get("returns_frame", cur_str) if rf is not None and len(rf) > 0: frames.append(rf) print(f"LENGTH FRAMES = {len(frames)}") return frames def compute_trend_scores(signalDate, rebalConfig, dframe=None): """ Returns per-asset momentum z-score (NOT the final multiplier). We'll turn it into a multiplier inside apply_alpha_conditioning so we can use the alpha sign to decide confirm vs conflict. """ if dframe is None or dframe.empty: PyLog.info("compute_trend_scores: dframe empty; skipping trend conditioning") return {} lookback = getattr(rebalConfig, 'trendLookback', 40) frames = get_returns_history(signalDate, lookback) if not frames: PyLog.info("compute_trend_scores: no returns history; skipping") return {} universe_assets = set(dframe['assetKey'].unique()) mfm = SignalMgr.getStatic('model_universe_frame') if mfm is None or mfm.empty: PyLog.info("model_universe_frame empty; skipping trend conditioning") return {} asia_assets = universe_assets.intersection(mfm['assetKey'].unique()) if not asia_assets: PyLog.info("compute_trend_scores: no Asia assets in trading universe") return {} # Daily matrix: rows=assets, cols=days hist = pd.concat( [f.set_index('assetKey')['logretd'] for f in frames], axis=1 ).fillna(0.0) hist = hist.loc[hist.index.intersection(asia_assets)] if hist.empty: PyLog.info("compute_trend_scores: hist empty after filtering") return {} mom = hist.sum(axis=1) z = (mom - mom.mean()) / (mom.std() + 1e-9) PyLog.info(f"compute_trend_scores: computed z-scores for {len(z)} assets") return z.to_dict() def compute_trend_scores_v1(signalDate, rebalConfig, dframe=None): if dframe is None or dframe.empty: PyLog.info("compute_trend_scores: dframe empty; skipping trend conditioning") return {} lookback = getattr(rebalConfig, 'trendLookback', 40) frames = get_returns_history(signalDate, lookback) if not frames: PyLog.info("compute_trend_scores: no returns history; skipping") return {} # --- True universe is today's dframe --- universe_assets = set(dframe['assetKey'].unique()) # --- Allowed countries from config --- allowed_countries = set(Universe.getModelCountries()) # Load ModelCntry mapping mfm = SignalMgr.getStatic('model_universe_frame') if mfm is None or mfm.empty: PyLog.info("model_universe_frame empty; skipping trend conditioning") return {} mfm = mfm.dropna(subset=['assetKey', 'ModelCntry'])[['assetKey', 'ModelCntry']] mfm = mfm[mfm['ModelCntry'].isin(allowed_countries)] # Intersection of trade universe and Asia universe asia_assets = universe_assets.intersection(mfm['assetKey'].unique()) if not asia_assets: PyLog.info("compute_trend_scores: no Asia assets in trading universe") return {} # Build daily matrix for only those assets hist = pd.concat( [f.set_index('assetKey')['logretd'] for f in frames], axis=1 ).fillna(0.0) hist = hist.loc[hist.index.intersection(asia_assets)] if hist.empty: PyLog.info("compute_trend_scores: hist empty after filtering") return {} # Trend signal mom = hist.sum(axis=1) mom_norm = (mom - mom.mean()) / (mom.std() + 1e-9) mult = (1.0 + 0.2 * mom_norm).clip(0.5, 1.5) PyLog.info(f"compute_trend_scores: computed for {len(mult)} assets") return mult.to_dict() def compute_country_vol_scalars(signalDate, rebalConfig, dframe=None): if dframe is None or dframe.empty: PyLog.info("compute_country_vol_scalars: dframe empty; skipping") return {} lookback = getattr(rebalConfig, 'volLookback', 40) frames = get_returns_history(signalDate, lookback) if not frames: PyLog.info("compute_country_vol_scalars: no returns history; skipping") return {} hist = pd.concat(frames) # Today's universe assets universe_assets = set(dframe['assetKey'].unique()) # Allowed Asia countries allowed_countries = set(Universe.getModelCountries()) # Load mapping mfm = SignalMgr.getStatic('model_universe_frame') if mfm is None or mfm.empty: PyLog.info("model_universe_frame empty; skipping vol conditioning") return {} mfm = mfm.dropna(subset=['assetKey','ModelCntry'])[['assetKey','ModelCntry']] mfm = mfm[mfm['ModelCntry'].isin(allowed_countries)] # Merge country info into returns hist = hist.merge(mfm, on='assetKey', how='left') hist['ModelCntry'] = hist['ModelCntry'].fillna("UNKNOWN") # Reduce to trade universe assets hist = hist[hist['assetKey'].isin(universe_assets)] # Compute vol per country country_vols = {} for c in allowed_countries: grp = hist[hist['ModelCntry'] == c] if grp.empty: continue vol = grp['logretd'].std() if pd.isna(vol): continue scalar = 1.0 / (1.0 + 5.0 * vol) scalar = max(0.4, min(1.0, scalar)) PyLog.info(f"compute_country_vol_scalars: {c} → {scalar}") country_vols[c] = scalar # fallback for unknown country_vols["UNKNOWN"] = 1.0 return country_vols def trend_multiplier(alpha: float, z: float, k: float = 0.20, min_mult: float = 0.40, max_mult: float = 1.50, z_cap: float = 6.0, short_tail_z: float = 2.5, short_tail_cap: float = 1.00) -> float: """ Base behavior (unchanged from original): - multiplier = clip(1 + k*z, min_mult, max_mult) Tail safety brake: - ONLY if alpha < 0 AND z >= short_tail_z - cap multiplier at short_tail_cap (default = 1.0) This preserves: - mean-reversion shorts - policy-regime profitability - long-side behavior And only protects against: - extreme short squeeze tails """ # Safety if alpha == 0 or not np.isfinite(alpha) or not np.isfinite(z): return 1.0 # Cap extreme z z = float(max(-z_cap, min(z_cap, z))) # Original logic m = 1.0 + k * z m = max(min_mult, min(max_mult, m)) # Extreme short squeeze brake if alpha < 0 and z >= short_tail_z: m = min(m, short_tail_cap) return float(m) def apply_alpha_conditioning(dframe, signalDate, rebalConfig): trend_z = compute_trend_scores(signalDate, rebalConfig, dframe=dframe) vol = compute_country_vol_scalars(signalDate, rebalConfig, dframe=dframe) # knobs (safe defaults) k = getattr(rebalConfig, 'trendK', 0.20) # sensitivity to |z| min_mult = getattr(rebalConfig, 'trendMinMult', 0.40) # floor max_mult = getattr(rebalConfig, 'trendMaxMult', 1.50) # cap z_cap = getattr(rebalConfig, 'trendZCap', 3.0) # avoid extreme z blowing things up count = 0 total = len(dframe) def _trend_mult_legacy_unused(alpha, z): if alpha == 0 or not np.isfinite(alpha) or not np.isfinite(z): return 1.0 # cap z to avoid crazy multipliers z = float(max(-z_cap, min(z_cap, z))) strength = 1.0 + k * abs(z) strength = max(1.0, min(max_mult, strength)) # strength is at least 1 # confirmation vs conflict if np.sign(alpha) == np.sign(z): m = strength # boost else: m = 1.0 / strength # shrink # hard clip final multiplier return max(min_mult, min(max_mult, m)) def _adj(row): nonlocal count count += 1 asset = row['assetKey'] initial_alpha = float(row['alpha']) z = float(trend_z.get(asset, 0.0)) #t = _trend_mult(initial_alpha, z) t = trend_multiplier( initial_alpha, z, k=k, min_mult=min_mult, max_mult=max_mult, z_cap=z_cap, short_tail_z=getattr(rebalConfig, 'trendShortTailZ', 2.5), short_tail_cap=getattr(rebalConfig, 'trendShortTailCap', 1.0), ) cntry = row.get('modelCountry', row.get('ModelCntry')) v = float(vol.get(cntry, 1.0)) adjusted = initial_alpha * t * v PyLog.info( f"apply_alpha_conditioning [{count}/{total}]: {asset} " f"alpha {initial_alpha:.6f} → {adjusted:.6f} " f"(z={z:.3f}, trendMult={t:.3f}, vol={v:.3f}, country={cntry})" ) return adjusted dframe['alpha'] = dframe.apply(_adj, axis=1) return dframe def apply_alpha_conditioning_v1(dframe, signalDate, rebalConfig): trend = compute_trend_scores(signalDate, rebalConfig, dframe=dframe) vol = compute_country_vol_scalars(signalDate, rebalConfig, dframe=dframe) # Running counter count = 0 total = len(dframe) def _adj(row): nonlocal count count += 1 asset = row['assetKey'] initial_alpha = row['alpha'] t = trend.get(asset, 1.0) # prefer 'modelCountry' if exists, else fallback cntry = row.get('modelCountry', row.get('ModelCntry')) v = vol.get(cntry, 1.0) adjusted = initial_alpha * t * v PyLog.info( f"apply_alpha_conditioning [{count}/{total}]: " f"{asset} alpha {initial_alpha:.6f} → {adjusted:.6f} " f"(trend={t:.4f}, vol={v:.4f}, country={cntry})" ) return adjusted dframe['alpha'] = dframe.apply(_adj, axis=1) return dframe # === End Conditioning Functions === def apply_stock_selection_alpha(rebalConfig, dframe, signalDate): alpha_col = rebalConfig.alphaName # # Output directory # logdir = os.environ.get("LOGDIR") or "/data/log" out_dir = os.path.join(logdir, "post_opt_results") os.makedirs(out_dir, exist_ok=True) PyLog.info( f"apply_stock_selection_alpha " f"optimizer_rows={len(dframe)} " f"countries={rebalConfig.stock_selection_alpha_countries}" ) countries = rebalConfig.stock_selection_alpha_countries if countries is None: raise ValueError( "stock_selection_alpha_countries cannot be None. " "Use ['ALL'] or a list of countries." ) countries_upper = [c.upper() for c in countries] # # Sanity checks # if "stock_selection_alpha" not in dframe.columns: PyLog.info( "apply_stock_selection_alpha: " "stock_selection_alpha not found. Skipping." ) return dframe if alpha_col not in dframe.columns: PyLog.info( f"apply_stock_selection_alpha: " f"{alpha_col} not found. Skipping." ) return dframe # # Save original alpha # original_path = os.path.join( out_dir, f"original_alpha_{signalDate}.csv" ) dframe[[alpha_col]].reset_index().to_csv( original_path, index=False ) # # Save production alpha + stock selection alpha # stock_selection_path = os.path.join( out_dir, f"stock_selection_alpha_{signalDate}.csv" ) dframe[ [alpha_col, "stock_selection_alpha"] ].reset_index().to_csv( stock_selection_path, index=False ) # # ALL => override every country # if "ALL" in countries_upper: mask = pd.Series(True, index=dframe.index) else: mask = ( dframe["modelCountry"] .astype(str) .str.upper() .isin(countries_upper) ) rows_updated = int(mask.sum()) # # Replace production alpha with stock_selection_alpha # dframe.loc[mask, alpha_col] = dframe.loc[ mask, "stock_selection_alpha" ] # # Drop intermediate column # dframe.drop( columns=["stock_selection_alpha"], inplace=True ) # # Save final alpha # final_path = os.path.join( out_dir, f"final_alpha_{signalDate}.csv" ) dframe[[alpha_col]].reset_index().to_csv( final_path, index=False ) null_alpha_count = int(dframe[alpha_col].isna().sum()) PyLog.info( f"apply_stock_selection_alpha " f"rows_updated={rows_updated} " f"optimizer_rows={len(dframe)} " f"null_alpha_count={null_alpha_count}" ) PyLog.info( "apply_stock_selection_alpha wrote files:\n" f" {original_path}\n" f" {stock_selection_path}\n" f" {final_path}" ) return dframe def apply_simple_alpha( rebalConfig, dframe, signalDate): # Output directory logdir = os.environ.get("LOGDIR") or "/data/log" out_dir = os.path.join(logdir, "post_opt_results") os.makedirs(out_dir, exist_ok=True) PyLog.info(f"apply_simple_alpha " f"alpha_names={rebalConfig.simple_alpha_names} " f"countries={countries}") alpha_col = rebalConfig.alphaName countries = rebalConfig.simple_alpha_countries if countries is None: raise ValueError( "simple_alpha_countries cannot be None. " "Use ['ALL'] or a list of countries." ) simple_alpha_df = get_simple_alpha( signalDate=signalDate, alpha_names=rebalConfig.simple_alpha_names, countries=countries, ) PyLog.info(f"simple_alpha rows={len(simple_alpha_df):,}") # Show total, positive and negative alpha counts country_stats = ( simple_alpha_df .assign( positive=lambda x: x["alpha"] > 0 ) .groupby("ModelCntry") .agg( total=("alpha", "size"), positive=("positive", "sum") ) ) country_stats["negative"] = (country_stats["total"] - country_stats["positive"]) PyLog.info("\nSIMPLE ALPHA COUNTRY COUNTS\n" + country_stats[ ["total", "positive", "negative"] ].to_string()) # # Save original alpha # original_path = os.path.join(out_dir, f"original_alpha_{signalDate}.csv") dframe[[alpha_col]].reset_index().to_csv(original_path, index=False) # # Save simple alpha # simple_alpha_path = os.path.join(out_dir, f"simple_alpha_{signalDate}.csv") simple_alpha_df.to_csv(simple_alpha_path, index=False) ########################################################################### # Replace Simple Alpha and remove stocks with no Simple Alpha ########################################################################### simple_alpha_df = simple_alpha_df.set_index("assetKey") optimizer_rows_before = len(dframe) common_assets = dframe.index.intersection(simple_alpha_df.index) # Replace alpha dframe.loc[ common_assets, alpha_col ] = simple_alpha_df.loc[ common_assets, alpha_col ] # Remove stocks with no Simple Alpha dframe = dframe.loc[common_assets] PyLog.info( f"apply_simple_alpha " f"optimizer_rows_before={optimizer_rows_before} " f"simple_alpha_rows={len(simple_alpha_df)} " f"matched_assets={len(common_assets)} " f"optimizer_rows_after={len(dframe)}" ) # # Save final alpha # final_path = os.path.join( out_dir, f"final_alpha_{signalDate}.csv" ) dframe[[alpha_col]].reset_index().to_csv( final_path, index=False ) PyLog.info( f"apply_simple_alpha" f"wrote files:\n" f" {original_path}\n" f" {simple_alpha_path}\n" f" {final_path}" ) return dframe def apply_3ai_alpha(rebalConfig, dframe, signalDate): from datetime import datetime, date ####################################################################### # Configuration ####################################################################### tradeDate = PyDate.nextWeekday(signalDate) if isinstance(tradeDate, date) and not isinstance(tradeDate, datetime): tradeDate_dt = datetime.combine(tradeDate, datetime.min.time()) else: tradeDate_dt = tradeDate trade_date_str = tradeDate_dt.strftime("%Y-%m-%d") alpha_col = rebalConfig.alphaName countries = rebalConfig.three_ai_alpha_countries if countries is None: raise ValueError( "three_ai_alpha_countries cannot be None. " "Use ['ALL'] or a list of countries." ) countries = [c.upper() for c in countries] pct = float(rebalConfig.top_bottom_pct_3ai_stocks) if pct <= 0 or pct > 100: raise ValueError( f"top_bottom_pct_3ai_stocks must be in (0,100], got {pct}" ) strategy_type = str(rebalConfig.strategyLongShortType).upper() if strategy_type not in {"LONG", "SHORT", "BOTH"}: raise ValueError( f"Unknown strategyLongShortType={strategy_type}" ) ####################################################################### # Logging directory ####################################################################### logdir = os.environ.get("LOGDIR") or "/data/log" out_dir = os.path.join(logdir, "post_opt_results") os.makedirs(out_dir, exist_ok=True) ####################################################################### # Save original optimizer alpha ####################################################################### original_path = os.path.join(out_dir, f"original_alpha_{signalDate}.csv") dframe[[alpha_col]].reset_index().to_csv(original_path, index=False) PyLog.info( f"apply_3ai_alpha " f"tradeDate={trade_date_str} " f"countries={countries} " f"pct={pct} " f"strategy={strategy_type}" ) ####################################################################### # Find latest 3AI file ####################################################################### base_dir = rebalConfig.three_ai_data_dir PyLog.info(f"Reading 3ai files from {base_dir}") if base_dir is None: raise ValueError( "three_ai_data_dir must be configured." ) pattern = re.compile( r"3ai_ALBAScoresv2\.3_(\d{4}-\d{2}-\d{2})\.csv" ) dated_files = [] for fname in os.listdir(base_dir): m = pattern.match(fname) if not m: continue file_date = datetime.strptime( m.group(1), "%Y-%m-%d" ) if file_date <= tradeDate_dt: dated_files.append((file_date, fname)) if len(dated_files) == 0: PyLog.info( f"{trade_date_str} | " f"No 3AI file found." ) return dframe dated_files.sort() selected_file_date, selected_file = dated_files[-1] age_days = (tradeDate_dt - selected_file_date).days if age_days > 8: PyLog.info(f"{trade_date_str} | " f"3AI file stale " f"(age={age_days})") return dframe file_path = os.path.join(base_dir, selected_file) PyLog.info(f"{trade_date_str} | " f"Using 3AI file " f"{selected_file}") ####################################################################### # Read vendor file ####################################################################### three_ai_df = pd.read_csv(file_path) three_ai_df["ISIN"] = ( three_ai_df["ISIN"] .astype(str) .str.strip() ) three_ai_df = three_ai_df[ [ "ISIN", "exp_alpha", "alpha_rank" ] ].copy() ####################################################################### # Build ISIN -> assetKey mapping ####################################################################### mfm = SignalMgr.getStatic( "model_universe_frame" ) mfm["RkdISIN"] = ( mfm["RkdISIN"] .astype(str) .str.strip() ) from model.domain.Universe import Universe model_countries = Universe.getModelCountries("univ_asia_v1") mfm = mfm[ mfm["ModelCntry"].isin(model_countries) ] trade_date_only = tradeDate_dt.date() mfm = mfm[ (mfm["startDate"] <= trade_date_only) & ( mfm["endDate"].isna() | ( mfm["endDate"] >= trade_date_only ) ) ] mfm = mfm[ [ "assetKey", "RkdISIN", "ModelCntry" ] ].drop_duplicates() ####################################################################### # Convert ISIN -> assetKey ####################################################################### three_ai_df = three_ai_df.merge( mfm, left_on="ISIN", right_on="RkdISIN", how="inner" ) PyLog.info( f"{trade_date_str} | " f"Mapped vendor rows = " f"{len(three_ai_df):,}" ) ####################################################################### # Filter requested countries ####################################################################### if "ALL" not in countries: three_ai_df = three_ai_df[ three_ai_df["ModelCntry"] .astype(str) .str.upper() .isin(countries) ].copy() PyLog.info( f"{trade_date_str} | " f"3AI rows after country filter = " f"{len(three_ai_df):,}" ) ####################################################################### # Apply top / bottom percentile selection ####################################################################### if pct < 100: frac = pct / 100.0 top_cut = 1.0 - frac bottom_cut = frac long_df = three_ai_df[ three_ai_df["alpha_rank"] >= top_cut ] short_df = three_ai_df[ three_ai_df["alpha_rank"] <= bottom_cut ] if strategy_type == "LONG": three_ai_df = long_df.copy() elif strategy_type == "SHORT": three_ai_df = short_df.copy() else: three_ai_df = pd.concat( [long_df, short_df], ignore_index=True ) PyLog.info( f"{trade_date_str} | " f"TopBottom={pct}% " f"LONG={len(long_df):,} " f"SHORT={len(short_df):,} " f"SELECTED={len(three_ai_df):,}" ) else: PyLog.info( f"{trade_date_str} | " f"Using all mapped 3AI names." ) ####################################################################### # Remove duplicate assetKeys if any ####################################################################### three_ai_df = ( three_ai_df .drop_duplicates( subset="assetKey", keep="first" ) .copy() ) ####################################################################### # Rename alpha column ####################################################################### three_ai_df = three_ai_df.rename( columns={ "exp_alpha": alpha_col } ) ####################################################################### # Save processed 3AI alpha ####################################################################### three_ai_path = os.path.join( out_dir, f"3ai_alpha_{signalDate}.csv" ) three_ai_df.to_csv( three_ai_path, index=False ) ####################################################################### # Country statistics ####################################################################### country_stats = ( three_ai_df .assign( positive=lambda x: x[alpha_col] > 0 ) .groupby("ModelCntry") .agg( total=(alpha_col, "size"), positive=("positive", "sum") ) ) country_stats["negative"] = ( country_stats["total"] - country_stats["positive"] ) PyLog.info( "\n3AI COUNTRY COUNTS\n" + country_stats[ [ "total", "positive", "negative" ] ].to_string() ) # Replace 3AI alpha three_ai_df = three_ai_df.set_index("assetKey") optimizer_rows_before = len(dframe) common_assets = dframe.index.intersection(three_ai_df.index) # Replace alpha dframe.loc[ common_assets, alpha_col ] = three_ai_df.loc[ common_assets, alpha_col ] # Remove stocks with no 3AI alpha dframe = dframe.loc[common_assets] PyLog.info( f"apply_3ai_alpha " f"optimizer_rows_before={optimizer_rows_before} " f"3ai_rows={len(three_ai_df)} " f"matched_assets={len(common_assets)} " f"optimizer_rows_after={len(dframe)}" ) ####################################################################### # Save final optimizer alpha ####################################################################### final_alpha_path = os.path.join( out_dir, f"final_alpha_{signalDate}.csv" ) dframe[[alpha_col]].reset_index().to_csv( final_alpha_path, index=False ) ####################################################################### # Summary logging ####################################################################### PyLog.info( f"{trade_date_str} | " f"3AI overlay complete | " f"optimizer_rows={len(dframe):,} " f"overlay_rows={len(three_ai_df):,} " f"common_assets={len(common_assets):,}" ) PyLog.info(f"Original alpha written : {original_path}") PyLog.info(f"3AI alpha written : {three_ai_path}") PyLog.info(f"Final alpha written : {final_alpha_path}") ####################################################################### # Return optimizer dataframe ####################################################################### return dframe