""" Series 3, Post 5: RLHF/DPO/GRPO/CAI Overview Radar chart comparison of 4 alignment methods 1080x1080, 25 sec, 10fps = 250 frames """ import numpy as np import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import matplotlib.patches as mpatches from matplotlib.patches import FancyArrowPatch import matplotlib.patheffects as pe from matplotlib.animation import FFMpegWriter import matplotlib.gridspec as gridspec # === CONFIG === FPS = 10 DURATION = 25 N_FRAMES = FPS * DURATION WIDTH, HEIGHT = 1080, 1080 DPI = 100 # === COLORS === BG = '#0d1117' GRID_COL = '#21262d' TEXT_COL = '#e6edf3' DIM_COL = '#8b949e' METHOD_COLORS = { 'RLHF': '#f85149', # red 'DPO': '#3fb950', # green 'GRPO': '#58a6ff', # blue 'CAI': '#e3b341', # gold } # === RADAR DATA === # Dimensions (higher = better) DIMS = [ 'Автономность\n(без людей)', 'Стабильность\nобучения', 'Вычислительная\nэффективность', 'Универсальность\nзадач', 'Качество\nвыхода', ] N_DIMS = len(DIMS) # Scores 0-5 for each method METHOD_SCORES = { 'RLHF': [1.0, 2.0, 1.5, 5.0, 5.0], 'DPO': [1.5, 5.0, 4.5, 4.5, 5.0], 'GRPO': [5.0, 3.5, 3.0, 2.5, 4.5], 'CAI': [5.0, 4.0, 4.0, 5.0, 4.0], } METHODS = list(METHOD_SCORES.keys()) # Timeline of method appearances # Frame ranges for each phase PHASE_INTRO = (0, 30) # Title PHASE_M1 = (30, 80) # RLHF appears PHASE_M2 = (80, 130) # DPO appears PHASE_M3 = (130, 180) # GRPO appears PHASE_M4 = (180, 230) # CAI appears PHASE_ALL = (230, 250) # All together, hold def make_radar_axes(ax): """Setup polar axes for radar chart.""" angles = np.linspace(0, 2 * np.pi, N_DIMS, endpoint=False) angles = np.concatenate([angles, [angles[0]]]) # close the polygon # Grid circles for r in [1, 2, 3, 4, 5]: circle_pts = np.full(100, r) theta = np.linspace(0, 2*np.pi, 100) ax.plot(theta, circle_pts, color=GRID_COL, lw=0.5, alpha=0.5) # Spokes for angle in angles[:-1]: ax.plot([angle, angle], [0, 5], color=GRID_COL, lw=0.7, alpha=0.7) return angles def draw_method(ax, angles, scores, color, alpha=1.0, label=None): """Draw a single method polygon on radar.""" vals = scores + [scores[0]] # close line, = ax.plot(angles, vals, color=color, lw=2.5, alpha=alpha) poly = ax.fill(angles, vals, color=color, alpha=0.15 * alpha) # Dots at vertices ax.scatter(angles[:-1], scores, color=color, s=60, zorder=5, alpha=alpha) return line def ease_in_out(t): """Smooth easing function 0->1.""" return t * t * (3 - 2 * t) def get_alpha(frame, start, end, hold_end=None): """Fade in alpha for a method.""" if frame < start: return 0.0 if frame < end: t = (frame - start) / (end - start) return ease_in_out(t) return 1.0 # Key labels and descriptions for each method METHOD_INFO = { 'RLHF': '3 модели: SFT→RM→PPO\n"как ChatGPT"', 'DPO': '1 функция потерь\n"без reward model"', 'GRPO': 'группа ответов → baseline\n"как DeepSeek R1"', 'CAI': 'конституция + self-critique\n"как Claude"', } METHOD_DATES = { 'RLHF': '2022', 'DPO': '2023', 'GRPO': '2024', 'CAI': '2022', } fig = plt.figure(figsize=(WIDTH/DPI, HEIGHT/DPI), facecolor=BG, dpi=DPI) fig.subplots_adjust(left=0.05, right=0.95, top=0.88, bottom=0.08) # Main radar chart (polar) ax_radar = fig.add_axes([0.12, 0.15, 0.76, 0.65], polar=True) ax_radar.set_facecolor(BG) ax_radar.set_theta_offset(np.pi / 2) ax_radar.set_theta_direction(-1) ax_radar.set_ylim(0, 5.5) ax_radar.set_xticks([]) ax_radar.set_yticks([]) ax_radar.spines['polar'].set_visible(False) ax_radar.grid(False) angles = make_radar_axes(ax_radar) # Dim labels for i, (angle, dim) in enumerate(zip(angles[:-1], DIMS)): x = np.cos(angle - np.pi/2) * 5.8 y = np.sin(angle - np.pi/2) * 5.8 # Convert to display coords ax_radar.text(angle, 6.2, dim, ha='center', va='center', fontsize=8.5, color=DIM_COL, fontweight='bold', multialignment='center') # Score labels on innermost/outermost ring for r in [1, 3, 5]: ax_radar.text(np.pi/2, r+0.1, str(r), ha='center', va='bottom', fontsize=7, color=GRID_COL) # Title title_ax = fig.add_axes([0, 0.88, 1, 0.12]) title_ax.set_facecolor(BG) title_ax.axis('off') title_ax.text(0.5, 0.65, '4 способа выровнять LLM', ha='center', va='center', fontsize=22, color=TEXT_COL, fontweight='bold', transform=title_ax.transAxes) title_ax.text(0.5, 0.15, 'RLHF · DPO · GRPO · CAI — сравнение @fminxyz', ha='center', va='center', fontsize=12, color=DIM_COL, transform=title_ax.transAxes) # Bottom legend area legend_ax = fig.add_axes([0.05, 0.02, 0.9, 0.13]) legend_ax.set_facecolor(BG) legend_ax.axis('off') # Pre-draw method lines (invisible initially) method_lines = {} method_fills = {} for name in METHODS: scores = METHOD_SCORES[name] color = METHOD_COLORS[name] vals = scores + [scores[0]] line, = ax_radar.plot(angles, vals, color=color, lw=2.5, alpha=0) ax_radar.fill(angles, vals, color=color, alpha=0) dots = ax_radar.scatter(angles[:-1], scores, color=color, s=60, zorder=5, alpha=0) method_lines[name] = (line, dots) # Phase bounds for each method METHOD_PHASES = { 'RLHF': PHASE_M1, 'DPO': PHASE_M2, 'GRPO': PHASE_M3, 'CAI': PHASE_M4, } # Legend boxes at bottom legend_items = [] for i, name in enumerate(METHODS): color = METHOD_COLORS[name] info = METHOD_INFO[name] year = METHOD_DATES[name] x = 0.12 + i * 0.23 rect = plt.Rectangle((x - 0.1, 0.05), 0.2, 0.85, transform=legend_ax.transAxes, facecolor=color+'22', edgecolor=color+'44', linewidth=1.5) legend_ax.add_patch(rect) legend_ax.text(x, 0.85, f'● {name}', ha='center', va='top', fontsize=11, color=color, fontweight='bold', transform=legend_ax.transAxes) legend_ax.text(x, 0.52, info, ha='center', va='center', fontsize=7.5, color=TEXT_COL, alpha=0.85, transform=legend_ax.transAxes, multialignment='center') legend_ax.text(x, 0.1, year, ha='center', va='bottom', fontsize=8, color=color, alpha=0.7, transform=legend_ax.transAxes, style='italic') legend_items.append(rect) # Store all artists for animation drawn_fills = {name: [] for name in METHODS} drawn_lines = {name: None for name in METHODS} drawn_dots = {name: None for name in METHODS} # Clear and redraw each frame def draw_frame(frame): ax_radar.cla() ax_radar.set_facecolor(BG) ax_radar.set_theta_offset(np.pi / 2) ax_radar.set_theta_direction(-1) ax_radar.set_ylim(0, 5.5) ax_radar.set_xticks([]) ax_radar.set_yticks([]) ax_radar.spines['polar'].set_visible(False) ax_radar.grid(False) make_radar_axes(ax_radar) # Dim labels for i, (angle, dim) in enumerate(zip(angles[:-1], DIMS)): ax_radar.text(angle, 6.2, dim, ha='center', va='center', fontsize=8.5, color=DIM_COL, fontweight='bold', multialignment='center') # Score refs for r in [1, 3, 5]: ax_radar.text(np.pi/2 + 0.05, r + 0.1, str(r), ha='left', va='bottom', fontsize=7, color=GRID_COL) # Draw each method with appropriate alpha for name in METHODS: start, end = METHOD_PHASES[name] alpha = get_alpha(frame, start, end) if alpha <= 0: continue scores = METHOD_SCORES[name] color = METHOD_COLORS[name] vals = scores + [scores[0]] ax_radar.plot(angles, vals, color=color, lw=2.5, alpha=alpha) ax_radar.fill(angles, vals, color=color, alpha=0.18 * alpha) ax_radar.scatter(angles[:-1], scores, color=color, s=60, zorder=5, alpha=alpha) # Label on chart if alpha > 0.5: # Find best position (highest score dimension) best_dim = np.argmax(scores) best_angle = angles[best_dim] best_r = scores[best_dim] label_r = best_r + 0.4 ax_radar.text(best_angle, label_r, name, ha='center', va='center', fontsize=9, color=color, fontweight='bold', alpha=min(1.0, (alpha - 0.5) * 2)) # Legend alpha for i, name in enumerate(METHODS): start, end = METHOD_PHASES[name] alpha = get_alpha(frame, start, end) legend_items[i].set_alpha(alpha * 0.7) writer = FFMpegWriter(fps=FPS, bitrate=2000, extra_args=['-vcodec', 'libx264', '-pix_fmt', 'yuv420p']) out_path = '/root/Strategy/content/drafts/rlhf_overview_animation.mp4' with writer.saving(fig, out_path, dpi=DPI): for frame in range(N_FRAMES): draw_frame(frame) writer.grab_frame() plt.close(fig) print(f"✅ Saved: {out_path}") print(f" Frames: {N_FRAMES}, FPS: {FPS}, Duration: {DURATION}s")