+ j5qdZddlmZddlZddlZddlZddlmZddlm Z ddl Z ddl Z ddl mcmZddlmZddlmZddlmZmZdd lmZmZmZd d lmZd d lmZd d l m!Z! ddl"Z"dZ#n #e$$rdZ#YnwxYwee%&j'dZ(e(dz dz Z)e(dz dz Z*e(dz dz dz Z+d-dZ,d.d Z-d-d!Z.d/d#Z/ d0d1d*Z0d+Z1e2d,kr e1dSdS)2uBV3 face training — distill teacher's .npz targets into a causal TCN. Usage: PYTHONPATH=. python3 -m models.v3_face.train # full train PYTHONPATH=. python3 -m models.v3_face.train --smoke # 10 scenarios × 5 epochs PYTHONPATH=. python3 -m models.v3_face.train --device cuda:1 --epochs 80 ) annotationsN)Path)Dict)AdamW)LambdaLR) DataLoaderWeightedRandomSampler) LIPSYNC_ONLYEXPRESSION_ONLYSHARED_CHANNELS) V3FaceConfig)BlendshapeDataset) V3FaceModelTFdata v3_trainingemotionmodelsv3_face checkpointscfgrreturn torch.Tensorctj|jf|jtj}t D] }|j||< tD]}d|j|jzz||<|j|d<|j|d<|S)zPer-channel L1 weight vector, shape (52,). Lipsync channels get cfg.lipsync_weight (audio-sync matters most). Eye-blink (ch 8, 9) get cfg.eye_blink_weight (sparse, hard to learn). Everything else gets cfg.expression_weight. dtype? ) torchfull output_dimexpression_weightfloat32r lipsync_weightr eye_blink_weightrwchs L/dataset/kemix-engine/package/face/animasync-face-v3/models/v3_face/train.pymake_channel_weightsr,+s  CN$c&;5=QQQA##""CCs)C,AAB"  AaD  AaD Hpredtarget valid_length ch_weightsc|j\}}}tj||jd}||dk}||z |ddddfz} | d} | |z} | d} | | z S) uL1 loss masked to valid frames, weighted per channel. pred, target: (B, T, C) valid_length: (B,) int — number of valid frames per sample ch_weights: (C,) devicerr Ndim?min) shaper!aranger4 unsqueezefloatabsmeansumclamp) r.r/r0r1BTC frame_idxmaskdiffdenoms r+ masked_l1rJ?sjGAq! Qt{333==a@@I  ..q11 1 8 8 : :D 6M   :dD!!!m#< rArB) r.r/r0rWrCrD_pred_vtarget_vrHrFrGrIs r+masked_velocityr\rs-jGAq! !!!QRR%[43B3< 'Faaae}vaaa"f~-H X  " " $ ${4qqq='A AD 999  D QU4;777AA!DDI )44Q77 7 > > @ @D $;D HHJJ    % %E 88:: r-r8trainboolvelocity_scaler>Dict[str, float]c ||ddddd} |rtjntj} | 5|D]} | d|d} | d|d}| d|d}| d |d}|| |}t ||||}t ||||| z}||z}|r|d |tj j | |j ||| d xxt|z cc<| d xxt|z cc<| d xx|jdkr!t|ndz cc<| dxxdz cc< dddn #1swxYwYt%d| d}| d |z | d |z | d |z dS)Nr)lossl1velnaudioT) non_blockingcondr/r0) set_to_nonercrdrerfr )rcrdre)r]r! enable_gradno_gradtorJr\ zero_gradbackwardnnutilsclip_grad_norm_ parameters grad_clipstepr>itemvelocity_weightmax)modeldl optimizer schedulerr1rWrr4r]r_sumsgrad_ctxbatchrgrir/r0r.loss_l1loss_velrcrfs r+ run_epochrs KKs3Q 7 7D&+@u """H   E'N%%f4%@@E=##F#>>D8_''T'BBF 033F3NNL5%%DflJGGG'tV\(3n(DFFHX%D !###555 ..u/?/?/A/A3=QQQ      LLLE$))++.. .LLL JJJ% // /JJJ KKKS5H15L5L5111RU UKKK IIINIIII3 8 AtCyAL1$DJN4;QR? S SSsG.II Icn$%&tj}|dtt|dtt |dtt |dtd|dtd|dtd|dd |d td |d dd|dgddd|dtdd|ddd|ddd|ddd |d!dd" |d#dd$ |d%td&d'|d(td&d)|d*tdd+|d,tdd-|d.tdd/|d0td1d2|d3td1d4|d5td1d6|d7tdd8|d9tdd:| }t$|j r |j $_ |jr |j$_|jr |j$_|j$_|j$_|j$_|j$_|j$_|j$_|j$_|j$_|j$_|jd;d;<t9j|j}$jd&ks$jd&ks$j;$j$jn$j}t=d=$jd>d?$jd>d@|d>dA$j"t=dB$jd>dC$jd>dD$j"t=dE$jd>dC$jd>dD$jd1krt=dF$jd>dG$jd1kr5t=dH$jd>dI$jdJz$j z dKdL$j dM$jd1kr5t=dN$jd>dI$jdJz$j z dKdL$j dM$j"t=dO$jd>dP$jd>dQtC|j"|j#dRz $j$$j$j$j$j$j$j$jS }tC|j"|j#dTz $j$$j$j$j$j$j$jU }|j%rg}dVD]5\&}&fdW|j&Dd|}|'|6|s|j&ddX}||_&|j&ddY|_&dY$_ dY$_(d $_t=dZtS|d[tS|d\$j d]$jt=d^tS|d_|*t=d`tS|d_|*|+$j,} t[| tS|d;a} t]|$j| |j/d;dbc} t]|$jdb|j/d;d} ta$1|} t=de| j2dfz d>dg| j3dhdi| j3d z dhdj|j4|j45stmdk|j4t9j7|j4|dbl}| 8|dmt=dn|j4do|9dpdqdr|9dstdtdudD|j:rB| :}t=dv|dfz d>dw| j;dfz d>dxdydzlot~}|j>rt~st=d{|r.|j@pJd|$jAd}t$d~r$jCntS$jDd$jd|j%rdndz}t}jE|jF|jG|i$jH| j2| j3| j3d z tS|tS||*|*t||j%d t}jI| ddt=dt|jJKd| LD}t|$jd$jN}tdytS| $j z%d$%fd }t||}t$1|}t$1|}dydld|!dhd| d|k}"|"r| d}| [$jH|| d| d|d|dd}|jTdkrdn d|jT}#|j\r |#d|j\}#|"r8t9j]||jd|#dz t=d|#d|dudDt9j]||jd|#dz |r]t}j^||d|d|d| d| d| d||Zd|!d |t=d|dut=d|j|r$|t|j_d<t}j`dSdS)Nz --npz_dir)typedefaultz --emotion_dirz --out_dirz--epochsz --batch_sizez--lrz--devicezcuda:0)rz --num_workersrOz--smoke store_truez)Smoke test: 10 train scenarios, 5 epochs.)actionhelpz--focus)alllipsync expressionrzChannel focus. 'all' = train all 52 channels (default). 'lipsync' = only LIPSYNC + SHARED channels have loss (expression branch sees zero gradient). 'expression' = only EXPRESSION_ONLY channels have loss (lipsync branch sees zero gradient).)choicesrrz--resumezvLoad model weights from a checkpoint .pt before training. Use with --freeze_lipsync for phase-2 expression retraining.)rrrz--freeze_lipsynczFreeze shared backbone + lipsync branch + lipsync head. Only the expression branch + head will train. Lipsync output stays bit-for-bit identical to what was loaded via --resume.z--wandbz5Log to Weights & Biases. Requires `wandb login` once.z--wandb_projectzanimasync-v3-facezW&B project name.)rrz--wandb_run_namez)W&B run name. Defaults to auto-generated.z--wandb_entityz4W&B entity (team or user). Defaults to your default.z--lipsync-target-gainr8zMultiply PURE_LIPSYNC target channels (jaw, mouth mechanics, tongue, cheekPuff) by this factor at load time, then clamp [0,1]. Default 1.0 = no change.z--expression-target-gainaRMultiply EMOTIONAL target channels (brows incl. innerUp, cheekSquint, eyeSquint, eyeWide, mouth Dimple/Frown/Smile, noseSneer) by this factor at load time, then clamp [0,1]. Default 1.0 = no change. If --emotional-mouth-target-gain is also set, THIS knob covers only the pure-expression subset (brows + eyeSquint + eyeWide + cheekSquint).z--emotional-mouth-target-gainuOptional separate gain for emotional-mouth target channels (mouthDimple, mouthFrown, mouthSmile, noseSneer). When set, decouples from --expression-target-gain so brows/eyes can go higher than mouth. Used by v18b to avoid pushing shared mouth channels past the point where crisp_mouth normalization destabilizes lipsync. None → same as --expression-target-gain (backward-compat with v14/v18).z--velocity-eye-squint-weightuPer-channel velocity penalty for eyeSquint L/R (ch 18, 19). Higher than the default expression velocity weight suppresses jitter at high gain. None → use velocity_expression_weight (backward-compat). Suggested 0.8 for v18b.z--velocity-brow-weightuPer-channel velocity penalty for the 5 brow channels (ch 0-4). Same mechanism as the eyeSquint weight — suppresses brow jitter at high expression gain. None → use velocity_expression_weight (backward-compat). Suggested 0.7 for v18c.z--plosive-damp-targetrba-Bake the runtime plosive damper into training targets. When mouthClose > 0.4 on a frame, mouthPress/Roll/Shrug (ch 35,36,39,40,41,42) get multiplied by (1 - this_value * smoothstep). 0 = off (default). 0.30 = matches the production main-viewer setting that prevents 'lips swallowed' on m/b/p plosives.z--smooth-target-sigma-browuGaussian σ (frames @ 30 fps) for pre-smoothing the 5 brow target channels BEFORE the gain. The proper fix for brow flicker at high gain — smooths the input the model is asked to fit so jitter never enters the training signal. 0 = off (default). Suggested 2.0 (~67ms) for v18e.z --smooth-target-sigma-eye-squintuGaussian σ (frames @ 30 fps) for pre-smoothing the eyeSquint L/R target channels BEFORE the gain. Real orbicularis oculi is slow + sustained, so we can smooth heavier than brows without losing useful motion. 0 = off (default). Suggested 3.0 (~100ms) for v18e.z--brow-innerup-happy-gainuqOverride the gain on browInnerUp (ch 2) for happy emotions (joy / laughter / excitement / gratitude). Set to 1.0 with expression-target-gain=2.2 to keep browInnerUp un-amplified on happy frames so the avatar doesn't look concerned/apologetic when saying happy things, while leaving the 2.2× boost everywhere else. None → no override (backward compat with v14..v18f).z --variant-taguOptional suffix appended to checkpoint filenames after --focus, so different gain runs don't clobber each other. E.g. 'v14' → best_lipsync_v14.pt / best_expression_v14.pt.T)parentsexist_oku[target-gain] lipsync×z.2fu expression×u emotional-mouth×z6 (applied per-channel in dataset, then clamped [0,1])u [velocity] eyeSquint(ch 18,19)×z (vs default expression )u[velocity] brows(ch 0-4)×z[plosive-damper] target damp=z, (applied in dataset when mouthClose > 0.4)u [target-smooth] brow(ch 0-4) σ=z frames (~iz.0fz ms @ z fps)u'[target-smooth] eyeSquint(ch 18,19) σ=z:[per-emotion] browInnerUp(ch 2) on happy frames uses gain z (vs z elsewhere)zseed_train_final.jsonl) crop_frameslipsync_target_gainexpression_target_gainemotional_mouth_target_gainplosive_damp_targetsmooth_target_sigma_browsmooth_target_sigma_eye_squintbrow_innerup_happy_gainzseed_val.jsonl)rrrrrrr))long_rO)solo_rN)z daily_-splitrNc,g|]}|dk|S)r ).0ewant_cats r+ zmain..^s'EEE1AaDH4D4DQ4D4D4Dr- z[smoke] train=z val=z epochs=z batch=ztrain: u scenarios — counts: zval: ) num_samples replacementF) batch_sizesampler num_workers pin_memory drop_last)rshufflerrzV3FaceModel (split-branch): g.Az M params, ~z.1fz MB fp32 (~z MB int8)z--resume checkpoint not found: ) map_location weights_onlyryz[resume] loaded weights from z (prev epoch=epoch?z , val_l1=val_l1nanz.4fz[freeze_lipsync] froze zGM params (shared backbone + lipsync branch + lipsync head). Trainable: zM (expression branch + head).r )LIPSYNC_BRANCH_CHANNELSuY[wandb] requested but `wandb` not installed — skipping. Install with: pip install wandbv3face_h_bn_blocks_lrz.0e_smoke) n_params size_mb_fp32 size_mb_int8train_scenarios val_scenariostrain_category_countsval_category_countsr4smoke)projectentitynameconfig gradientsd)loglog_freqz[wandb] logging to c g|] }|j | Sr) requires_grad)rps r+rzmain..sIIIaIIIIr-)g?gffffff?)lrbetas weight_decayruintrr>c |jkr|tdjz S|jz tdjz z }ddtjtjt d|zzzS)Nr rr8) warmup_stepsrxmathcospir:)ruprogr total_stepss r+ lr_lambdazmain..lr_lambdasv #" " "#a!1222 2s''3q+@P2P+Q+QQcDHTWs3~~%=>>>??r-)rEXPRESSION_BRANCH_CHANNELSrz[focus=lipsync] masked z expression channelsrz[focus=expressionz/freeze_lipsyncz ] masked z lipsync channelsinfr)r]r_zepoch 3dz train l1=rdz vel=rez val l1=z lr=z.2ez vw=z s)ryrrrval_veltrain_l1 train_velrYbestz.ptu → saved bestz (val l1=latestrc) rztrain/l1ztrain/velocityz train/losszval/l1z val/velocityzval/lossz val/best_l1r epoch_seconds)ruz Done. best val l1: z checkpoints: best_val_l1)rurrr>)aargparseArgumentParser add_argumentrDEFAULT_NPZ_DIRDEFAULT_EMOTION_DIRDEFAULT_OUT_DIRrr>str parse_argsrepochsn_epochsrr learning_raterrrrTrUrrrrout_dirmkdirr!r4printrPfpsrnpz_dir emotion_dirrrentriesextendrlencategory_countsget_sample_weightslong_oversample_weightr rrrrmrsize_mbresumeexists SystemExitloadload_state_dictgetfreeze_lipsync n_trainableryrwandb_WANDB_AVAILABLEwandb_run_name hidden_dimhasattrr dilationsinit wandb_project wandb_entity__dict__watchrunget_urlrsrrrxrr,rVrfocusrangetimevelocity_warmup_epochsr:r get_last_lr state_dict variant_tagsaversummaryfinish)'apargsr4mouth_gtrain_dsval_ds smoke_picksrfpickssample_weightsrtrain_dlval_dlryckptfrozen_nrr* use_wandbrun_nametrainable_paramsr{rr|r1rWrbest_valrt0r_trvadtis_bestsuffixrrrs' @@@r+mainr.s  " "BOOKdOODDDOOO$8KOLLLOOKdOODDDOOJS$O777OONdO;;;OOFO555OOJO111OOO#qO999OOIlDFFFOOI'G'G'G!EFFFOOJT42333OO&|4555 OOIlPRRROO%/B,...OO&DFFFOO$dOQQQOO+%#$$$ OO.UCOPPPOO3%GHHHOO2<=== OO,5$FGGG OO+%.///OO0uc;<<<OO6UC.///OO/eT3444OOO#tIJJJ ==??D ..C {.4;CL 8 w+DG!"6C!%!CLtd333 \$+ & &F 3&&)S00.:5A222  F(?GFF 7?FF")1FFF G G G %1 N1OWNN(+(FMNNN O O O + N3+CKNN(+(FMNNN O O O $$ A>]696M;>;WADAc:=:U W W WHt| $ 03C C,/O474K7:7Qs8}}>>3v;;>> >>-0^>> ? ? ? VCMM V V(:R:R:T:T V VWWW RCKK R R8N8N8P8P R RSSS001KLLN#N14X15777G(s~w&*&64$)+++H3>5$($4GGGF      ' 'E  K); K K Km J K K,1M!O J K K KLLL {{!!## NLt{LLMM Mz$+FOOO d7m,,, @dk@@!XXgs33@@5<<88?@@@ A A A   '')) U CUU!-c1TUUU V V V 322222)  B  //I z1*1 0 1 1 1;&  *s~ * *j9Q9Q)iWZ[^[hWiWi * *# ) * *:-xx2 /   &$ , !N % %  1#&x==!$V)1)A)A)C)C'-'='='?'?f++        "  E{S9999 9EI$5$5$7$799::: JI5#3#3#5#5III&*'#555I aX566K@@@@@@@ I..I%c**--f55J',,//77KKJJJJJJJ zY, " "B JrN!KOO ],F(G(G]]]^^^^ | # #t': #) " "B JrN!KOO Ht7J"R"3"3PRHH344HHH I I IU||Hs|$$<< Y[[)A-- US77 8 8 836 uh 9!;V4&4666ufi!;V5&4666Y[[2  u#T(>-/Y>4<+-e9<))++A.6#*  l    T(X%  $xH%%''lh%y4E   zU**0@DJ0@0@   433!133F  G JtT\,>6,>,>,>> ? ? ? EVEEhEEEE F F F 4(<(<(<(<<===   ItH"$U) jT( "5 vJ'++--a0!#       0( 0 0 0111 ($, ( ()))'/ m$ r-__main__)rrrr) r.rr/rr0rr1rrr) r.rr/rr0rrWrrr)r8)r]r^r_r>rr`)3__doc__ __future__rrrrpathlibrtypingrnumpynpr!torch.nn.functionalrp functionalF torch.optimrtorch.optim.lr_schedulerrtorch.utils.datarr scripts.compiler.constantsr r r rrdatasetrryrrr ImportError__file__resolver PROJECT_ROOTrrrr,rJrVr\rr.__name__rr-r+rCsb#"""""  ------>>>>>>>>UUUUUUUUUU &&&&&&LLLtH~~%%''/2 '-7"V+i7)I5 E     (&    @(