o h` @s*ddlZddlZddlZz ddlmZdZ Wn e y#dZdZ Ynwddl Z ddl Z ddl mZddlmZmZmZmZmZmZmZmZmZddlmZmZe jded z ejd d d Wne yrZzWYdZ[ndZ[wwGd ddZ ddZ!ddZ"ddZ#ddZ$e%dkre"dSdS)NTF)Counter) RandomMABEpsilonGreedyMABUpperConfidenceBoundMABThompsonSamplingMAB ContextualMABVotingEnsembleMABWeightedEnsembleMABDynamicEnsembleMABExpertEnsembleMAB)RecommenderLoggerperformance_trackerignore)categorygymnasium_env/RecommendationMABz@rl_recommender.UnifiedRecommendationEnv:UnifiedRecommendationEnv)id entry_pointc@sheZdZdZdddZddZddd ZdddZe ddddfddZ ddZ dddZ ddZ dS) MABSimulationz&Framework for running MAB experiments.epsilon_greedyNc Csz||_i|_tdd|_d|_||_|pi|_||jjdt |j t |ddt |ddt |ddd||d d dS) NZmab_simulation)namez$MAB Simulation framework initializedmax_armszN/An_users n_suggestions)rrr)Zenvironment_typeZenvironment_configalgorithm_typealgorithm_paramsextra) environmentresultsr loggercurrent_algorithmrr_initialize_algorithminfotype__name__getattr)selfrrrr(f/Users/divyeshpatel/Desktop/sahana/Recommender/recommender_rl/recommender/rl_recommender/simulation.py__init__>s      zMABSimulation.__init__c Cs zt|jdd}t|jdd}t|tr|dkrd}t|tr#|dkr%d}|jd|d||jdkr=||||_nS|jd krR|j d d }t |||d |_n>|jd krg|j dd}t |||d|_n)|jdkrst |||_n|jdkr|j d d}t |||d |_nt ||d d |_|jjd|jd|jjj||ddWdSty}zQ|jdt|z|dd|_|jdWn)ty}z|jdt|ddlm}|dd|_WYd}~n d}~wwWYd}~dSWYd}~dSd}~ww)z/Initialize the default algorithm based on type.r2rrz#Initializing algorithm with n_arms=z, n_suggestions=randomrepsilon皙?r.Zucbc@r1ZthompsonZ contextual皙?z Initialized z algorithm)Zalgorithm_classn_armsrrzError initializing algorithm: z*Fallback to RandomMAB algorithm successfulz Fallback algorithm also failed: )r r N)r&r isinstanceintr r#rr!rgetrrrr __class__r% Exceptionerrorstrrl_recommender.mabalgorithmsr)r'r5rrr.r1eZfallback_errorr(r(r)r"UsT       z#MABSimulation._initialize_algorithmcCs||_|pi|_||jS)z'Set a new algorithm for the simulation.)rrr"r!)r'rrr(r(r) set_algorithms zMABSimulation.set_algorithmr6r Tc+sPt}|jjt|||||di}|D]u\}} |r*|jd|dg} g} g} g} g}d}t| trog}gg}g}g}t |D]}| g}g}g}g}g}d}t |D]q}|j \}}| d}d}t |D]Wt jjd|j jdd} | | }!t|!dr|!nt|!}!|j |!\}}"}}}#|# d g}$| |!|"gt|!||"||"7}||||!||$||"7}qt||q`|||||||||qJt j|dd } t jdd } t j|dd } g} t t|dD]#fd d |D}%d d |%D}&t|&ddd}'| |'qg}g}(|D]})|(|)q;t t|D]fdd |(D}$|t j|$qJt fdd t |D}n| t |D]}|j \}}| d}d}t |D]dkr|j |!\}}"}}}#|# d g}$t| tr| j|!|"gt|!|dn | |!|"gt|!t jj|j j| jdd} t| tr| || }!n| | }!t|!dr|!nt|!}!|j |!\}}"}}}#|# d g}$t| tr| j|!|"gt|!|dn | |!|"gt|!| |"||"7}| || |!||$||"7}q| |qw|ri|dtd|ddkri|jd|dd|dt | d| | | | | |||||| j !t j"| j#| j t $| j#| j dkdd ||<q||_%t|}*|jj&||*t|||d|S)aN Run MAB experiment comparing different algorithms. Args: algorithms (dict): Dictionary of algorithm_name -> algorithm_instance n_users (int): Number of users to simulate n_steps (int): Number of recommendations per user random_runs (int): Number of random runs for each algorithm verbose (bool): Print progress ) algorithmsrn_steps random_runsverbosezRunning z algorithm...ruser_idr6)sizetolist event_types)axiscg|]}|qSr(r().0Zrun_selectionsstepr(r) z0MABSimulation.run_experiment..cSsg|] }|D]}|qqSr(r()rLZsublistitemr(r(r)rOr crKr(r()rLeventsrMr(r)rOrPcs g|] }|r|dqS)r()rLrun)cumulative_rewards_all_runsr(r)rOs )contextFrGreplacez Completed /z users, avg reward: .4f)outwhere) algorithm total_rewardscumulative_rewardsarm_selections user_rewardsrIfinal_total_rewardaverage_reward arm_countsZarm_avg_rewards)rdurationZ n_algorithmsrrC)'timer log_simulation_startlistkeysitemsr#r7rrangeresetrr9npr-randint n_productsselect_multiple_armshasattrrHrNupdatelenappendextendmeanr most_commonchoicerr5maxrfcopydivide arm_rewards zeros_likerlog_simulation_complete)+r'rBrrCrDrE start_timeralg_namer_r`rarbrcZevent_types_allcumulative_rewardZtotal_rewards_all_runsZarm_selections_all_runsZuser_rewards_all_runsZevent_types_all_runsrUZtotal_rewards_runZcumulative_rewards_runZarm_selections_runZuser_rewards_runZevent_types_runZcumulative_reward_runZuser_idxobs_rFZuser_total_rewardinput_arms_idxarmsrewardr#rIZstep_selectionsZflat_selectionsryZall_event_typesZrun_event_typesrgr()rVrNr)run_experiments                                   "* "  zMABSimulation.run_experimentc sz||_|js |||dur|j}|durt|j}t|t|}t |t r0| ||n| |t|krBd|n%t|krgfdd|D}|rgtj j ||tdd}t|gttd||jjd|d||jjtt|dd WSty} zE|jjd t| ||r|jjnd t| d d |durt|d krtj j |t|pdt|dd} | WYd} ~ SgWYd} ~ Sd} ~ ww)a Invoke the algorithm to get recommendations for a user. Args: algorithm: MAB algorithm instance event_df (pd.DataFrame): DataFrame containing recent user feedback events input_arms_idx: Available arms to choose from (if None, uses all arms) obs: Contextual information for contextual algorithms user_id: User identifier for logging n_recommendations: Number of recommendations to return (defaults to algorithm.n_suggestions) Returns: list: List of recommended item indices Ncsg|]}|vr|qSr(r()rLarmrecommendationsr(r)rO}sz.MABSimulation.invoke_agent..FrXz#Generated recommendations for user : )rFrr_n_recommendationsZinput_arms_countrzError in invoke_agent: None)rFr_r<rr,)r!emptyupdate_algorithmrroaranger5minrur7rrrr-rz concatenaterjsetr r#r:r%r;r<r=rH) r'r_event_dfrrrFrZremaining_arms additionalr?fallbackr(rr) invoke_agentOsP         zMABSimulation.invoke_agentc Cs|jr |jddS|D]\}}z|d}|d}|d}|dtj}ddd d d }||d } t|d r|j durt |t t j frOt |nt t|} d| kra|j jkrnn0|j | g| g|jjd|d|d|d| |||| | t|dr|nt|ddn|jd| d|j jdn|jdWqty} z&|jjd|dt| t|dr|nt|t| ddWYd} ~ qd} ~ wwdS)z Update the algorithm based on event data. Args: event_df (pd.DataFrame): DataFrame containing user feedback events Expected columns: user_id, item_id, event_type, timestamp z)No events to process for algorithm updateNrFitem_id event_type timestampr/gQ?g?g)viewcartpurchaseremove_from_cartgr!rz#Updated algorithm with event: user=z, item=z, event=z , reward= isoformat)rFrrr arm_indexrrz Arm index z" out of bounds for algorithm with z armsz$No current algorithm set for updatesz$Error updating algorithm with event rto_dict)eventr<)rr r#iterrowsr9pd Timestampnowrsr!r7r8rointegerr=r5rtrwarningr;r<r) r'rrrrFrrrZreward_mappingrrr?r(r(r)rsL     $  zMABSimulation.update_algorithmr6Fmab_experiment_results.pngcsts jddSjsjddStjdd|d\}}|d}jD]\}}|j|d|dd q(|d | d | d | |j d dd|d}t j} fdd| D} |j| | dd} |d| d|jddd| dt| | D]\} } |j| | d| d| ddddq|d}tj| d d!}t|}dt| }tjD]\}\}}|j||||d!|d|d"q|d#| d$| d%|||t| d&d|d'dt|D| |d(}d)djD}|j|| d*| d+| d,|jddd|j d ddt|rFt |dSt!dS)-z( Plot comparison of MAB algorithms. z/Matplotlib not available; skipping plot_resultsNz,No results to plot. Run an experiment first.)figsize)rrrag?)labelalphaz Time StepszCumulative RewardzCumulative Rewards Over TimeT333333?)r)rr csg|] }j|dqS)re)r)rLrr'r(r)rOrRz.MABSimulation.plot_results..ffffff?Z AlgorithmszAverage Rewardx-)rJrotationzAverage Reward per AlgorithmgMbP?r\centerbottom)hava)r rrrf)widthrrz Arm IndexzSelection CountzArm Selection Frequencyr cSsg|]}|qSr(r()rLir(r(r)rO s)r r cSsg|]}|dqS)rcr()rLresultr(r(r)rOrP) tick_labelszReward Distribution per UserzTotal Reward per User)"MATPLOTLIB_AVAILABLEr rrpltsubplotsrlplot set_xlabel set_ylabel set_titlelegendgridrjrkbar tick_paramsziptextget_x get_width get_heightruror enumerate set_xticksset_xticklabelsrmvaluesboxplot tight_layoutsavefigshow)r'rsavefilenamefigaxesaxrrZ alg_names avg_rewardsbarsrvaluer5rrrZuser_rewards_datar(rr) plot_resultss`        2  "      zMABSimulation.plot_resultscCs"|js |jddS|jd|jd|jdt|jdddd }t|d D]Z\}\}}|jd |d ||jd |dd|jd|dd|jdt|ddt |dddddd}|jd|d|d|dq.|jddS)z Print summary statistics. z1No results to summarize. Run an experiment first.Nz= ============================================================zMAB SIMULATION RESULTS SUMMARYz<============================================================cSs |ddS)Nr rer()rr(r(r)*s z-MABSimulation.print_summary..T)keyreverser  z. z Average Reward: rer\z Total Reward: rdz.2fz Std Dev (per user): rcrfrTz Top 3 Arms: z (counts: )) rr rr#sortedrlrrostdargsort)r'Zsorted_resultsrankrrtop_armsr(r(r) print_summarys     $zMABSimulation.print_summary)rN)N)r6rAr T)rFr)r% __module__ __qualname____doc__r*r"r@rr DataFramerrrrr(r(r(r)r;s  2 AM : FrcCst||t||ddt||ddt||ddt||g}t||t||ddt||t||ddt||ddt||ddt||ddt||ddt||d dt|||d d t|||d d t|||d d t|||ddt|||dddt |||ddd}|S)z,Create all available algorithms for testing.r/r0rr2r3r4g?rg?majorityrB voting_methodweightedrankedg{Gz?)rBweight_update_raterAg?)rB window_sizeswitch_threshold)rB learning_rate)Randomz UCB (c=2.0)Thompson SamplingzContextual MABuEpsilon-Greedy (ε=0.1)uEpsilon-Greedy (ε=0.3)uEpsilon-Greedy (ε=0.5)uEpsilon-Greedy (ε=0.7)uEpsilon-Greedy (ε=0.9)zVoting Ensemble (Majority)zVoting Ensemble (Weighted)zVoting Ensemble (Ranked)zWeighted EnsemblezDynamic EnsemblezExpert Ensemble) rrrrrrr|r r r )r5rZbase_algorithmsrBr(r(r)create_all_algorithms9s.          rcCsdd}d}d}d}tjdd||dd}t||}t|}|j|||d d }||jd d d d dS)z)Run comprehensive test of all algorithms.rAr,rZdrdataset/filtered_data.csv*datarrseedTrrCrE zcomprehensive_mab_results.pngrrrNgymmakerrrrrr5rrrCZmab_envrB simulationrr(r(r)run_comprehensive_test[ r cCsd}d}d}d}tjdd||dd}|jd t||t||d d t||d d t||t||t||t||dd t||d d gddd}t |}|j |||dd}| dS)z$Run test with visualization enabled.rr rArrrrhumanr4r0r2r3r/rr)ruEpsilon-Greedy (ε=0.2)ZUCBrzVoting EnsembleTrN) r r  unwrappedZset_render_moderrrrrrrrr r(r(r)run_visualization_testxs4       rcCsdd}d}d}d}tjdd||dd}t||}t|}|j|||d d }||jd d d ddS)z!Run quick test of all algorithms.r6rr,r+rrrrFrrTzquick_mab_results.pngrNrr r(r(r)run_quick_testrr__main__)& gymnasiumr numpyropandasrmatplotlib.pyplotpyplotrrr;warningsrh collectionsrr>rrrrrrr r r r r rfilterwarnings UserWarningregisterr?rrr rrr%r(r(r(r)sJ   ,   "'