Variational Auto-Encoders (VAE)

Transcription

1 Variational Auto-Encoders (VAE) Jonathan Pillow Lecture 21 slides NEU 560 Spring 2018

2 VAE Generative Model latent ~z N (0, I) data f <latexit sha1_base64="z+vg35/w+jutrabd7h0ejs+88ma=">aaacyhicdzhnbhmxemed5assxy0cuvhesfyidiskofb0ungoiktasnk0cpzzxoq/zm+mjsxfeasu8as8ew+dk64qswaks3/n7z9jj2dipfbyfl862z279+4/2huyp3r85omz/ypn5940jkofg2ncymi8skghjwildkwdpiyslibz4xw/widzwugvulqwuuxki1pwhik1qmcvzgdzel9b9ip10f1rtqjl2jgbh3r+vlpdgwuauwted8vc4igwh4jlihnvelcmz9kvdjputiefhfwdi32dmlnag5eorrro/l0rmpj+qsbjqrjo/dzbjf/fhg3wh0zbansgah57ud1iioaupqdt4ycjxcbbubpprztpmgmc0x/leaxhmhulmj6gsuhfdjfhbrnjjkbnywixfrob0xuuhdpegg+bnqnhjusyqjr1d2umhzun/ljwv55tpxjftcqzdkfbehgt6ai4vdkg8xbmy562w27vclech/bkold+ftc9+tjuey+8jk/ig1ks9+sinjaz0iecspknfcc/sk+zza6z5a0167q1l8hgzf9/a5t+5qk=</latexit> <latexit sha1_base64="aym7tr2bg/cxp6vzrb5v/5acui8=">aaac2hicdzhlbhmxfiad4vags1oq2lcxijckbnfmhqtlcjawqype2opmgnkct2pfn9lnqooxxa6x5rfyai/e2+bjr4gkccrlv873+/jyl0zwb1n2q5ncunzl6rwt6+mnm7dub3d37hw5xvvkblqlbu9k4pjgig2ag2anxjiis8goy9nlhh/pmxvcq7ewmgwkyznifaceymvcvvfmgcufcog4xmwblbvz49ep8ljby/rzsvcmyfvrq20djnc634ujprvkcqggzg3zzmdiewucchbsonbmedojz2wypsksuzffpidgh7ezwzw2csnay+7fozyrzi1kgz2swnsts6b5lzasoxo+8lyzgpiifwdvtccgcfmbemitoyawurbqebwrplnicyx4z2lakpaeaimjmvicq3nwp/5jhsiqaguc96cfabpiergsuc5ox1ztwrgpqiivqjjf58hvbqz6y4h/emjektcvput4g3u8p4+0ifr8hcxamatptddfz3jtho3186yfv3na23/r5ryf7qmhabfl6bnar6/qirogij6ib+gh+pm8sz4ln5mvf9ak0+65i1yq+fobzehprw==</latexit> ~ z ~x = f (~z ) + ~ ~x <latexit sha1_base64="f7a4yifurwl/dwic8w6hpefrwmg=">aaacxnicdzhnbhmxemed5assxy0cuvhesfyidiskofb00lnvjjjwyqav15lttpyxsmftppylhoerpajvxnvgtveijdcspb/m95+xx1pwujjmsl+95m7de/cf7dxmhz1+8vtz7t7zkton5tdkrhp7vjihumgyokajz7ufpkojp+x8sownc7bogp0zlzvmflvuohkcyuynigvwenox288g2srotsg70sddnfzs9x4wu8mbbrq5zm6n86zgiwcwbzcq0qjxudm+z5cwjlizbw7iv88n9hxmtgllbdwa6sr7d4vnyrmlkqntmzy5tdym/8xgdvyfjl7oukhq/paiqpeudw1np1nhgancrsg4ffgtlm+yzrzjd6vpoegkg6wynvpc6exw5/5thsi6qfsh788lnhwiv1c0tlvaofi3gq0zwqvfvlg/z4pf32r0x4l/9rxbfxk7kwfsh2/ixxwk7qqvn8g8a/oqxu3mm7vcfqp9qz4n8k/v+gcfuz3vkjfkfxldcvkehjajckkghjmv5bv5tn4kr4lomutq1pr0upoxzc2sr78b5k3j8g==</latexit> <latexit <latexit sha1_base64="ufamb61cbqdk8cpcyuhfvjid7va=">aaacxnicdzhnahsxemfl7ve6/uqayy+iptblzw4itmeqxhikcdrowosertwbq9bhis06mulqr+g1eys+ud+msrou2m4hbh/m95+rrlpwujjmsl+d5nhjj0+fbt1px7x89frn9s7bgton5ddnrhp7utihumjoo0ajf7ufpkoj5+x0amhpz2cdmporzmsykxatrsu4w5gafdpg9pzqu5v1smxqtzg3okvaol3a6fwsxoy3cjryyzwb5lmni88sci4hpexjogz8yq5hgkvmctzil58b6ieygdpk2hg00mx27wrplhnzvuanyjhx62yr/bcbnlh9gxmh6wzb84elqkzsnhqxox0lcxzlpargryhvpxzclomyfyhncw033cjf9ngxqs+cv/sf8hbwado6eh9zokldvikizdrvxsgqzwiyycr4oor9fr783kajpxb8r+feqsgxk3im/ck6nt1gulzhopi2ybrsun18fzebyrdxy7nefrbfpths97xf3ph35cpjywdyqi7jkekttr6rh+so3cfhiu6a5obbmntaml2yesn33+gt4/a=</latexit> <latexit <latexit sha1_base64="s3/+j2ddixcbzxvsirg0ntctugw=">aaac63icdzhlahrbfizr2ltsb4ku3rqoqoi4dadbn0lqjasqwukc05ohuuz0ppi6nfwnx4xfp4mld+lwjxu3+hy+jdwdrpwzpvdw839/nbqcvjtcyzl86kvxrl67fmprznzr9p2797z37h87u1koq26ksac5cycfhieklhbawmaql3csz183/gqb1gmj3+gyhlfi51ougjmm1mr7l1sapxf0js0mgc4a2w7rfnijt2irmiidke22nwystuimsdvrj10dtxz637kp4zucjvwy50zpuulym4ucs6jjrhjqmj5n5zakujmfbuzbn9x0cxcmtda2li20df/e4zlybqnykfqmz26dnea/2kjc4sxyc11wcjpfhlrukqkhzqfrqbdaus6dynykcffkz8wyjueb4zjt8j4bpzie+kzore3p/no0rlcb2rl2/ixdu9bhciqwavcab6sxo2emi9pnrejv09rvbzt6e8h/zg6tcrx5kwfsh67jxuwg7ztxwbwd8zoo003xz7kpjvchatji3z7rh7zq5rxfhpjhzjek5dk5ig/ierkstj6s7+qh+rmp6fp0ofpygy163z4hzkwir78bwzdxmw==</latexit> N (0, 2 I) <latexit sha1_base64="h578i619fhvo+geqdtk0pxojdew=">aaac43icdzfni1mxfibt69d4/ziolt1eizcclnulomtbn7ozrraza71tsdnz29b8ketwksfrf+7erx9acku/xx9jbucittudgtfnezotkzprnfmxzb9ayzwr167f2luz3rp deqvwvodcniitzpiewojpqd8xxoncgijhwojssxtf8bangmixfu5wgosazyupgiyupcfthadoyriqulbo4odbimhtab2acjhr47znxu5n1s3xgxze3oooaobkftl4vu0uradjrtqwd5jl2q0+my5rdsivkgiz0qwywifisaxbo170e/dhmprhuji7p8dr79wlphlurmyloqdzcbrm6+s82qfz5cuiz1judss8llrxhtuh6y/cugacor6ig1ld4vkznxbdq4velashha1vcedn1bzpl4ef+wr7cjnbgb+9hhvm6xblygsktvhy2burcnjfbf2w83+fb93yu+mnx//xewuved0oj98fbehkr6riovtggiwysqhqnm2/pclec9rp51s3fpe8cvwrmviceoefoeoxobtpcb9aj6iokpqlv6af6mudykfmcflm0jq3mzh20ecnx3+dc7fq=</latexit> <latexit sha1_base64="esjwap714kskhav2a3yyx5aurdo=">aaacyxicdzhbbhmxeiad5vswuwux3fhesewcaldcopcv3icqqikrtiibvl5ntrhiw9aedqmwr3gebuefecpebiddizlasjz+zffp2ompaykcztmvtnlt+o2bt7zup3fu3rv/yhvn4bezjexq50yae1oyb1jo6knacae1bazkcsfl5m2cn0zbomh0r5zxmftsqotkciyx9ak4tgo3e/7u2fl2n+tly6cbim9fl7rxdl7t+vmmdg8uaossotfisxqhnlkuxejii8zbzfiexcagss0uukffvjjqpzezopwx8wiky+zffz4p5+aqje7fcozw2sl5lzzosnofeqhrbkhzq4uqrli0dde+hqklhou8csatig+lfmws4xg/ku0ldz+5uyrpks+engz/5l/kiawcthxw/qxau4d4buxltkung1wb0tcwvfbfffv7ppi9juz/lphft9xq5itjozp+cb1pz5fogdpzopi0yblsun18fzeb4nivl2e9/mpl7shrds9b5df5qnzjtl6ra/kwhje+4usrb+q7+zg8ty6twfllypp02pphzcwsr78bduzkww==</latexit> N (0, I) <latexit sha1_base64="r17ca4yoky+34dtrjcgzhp0pjj8=">aaac5hicdzhlbhmxfiad4vags1nysreaiausrdmreiwr2mcmkhjpk2wsyon4eiu+jowzochyngu7xjyhymewxow3wzooeengsjz+ne/38exps8etjmmvvntt+o2bt/zux3fu3ru/3z54cgz1zsgbuc20ucijzyirngaogl2uhhgzc3ael17v/hzjjovavynvyuaszbqvocuqwpp2yxzizleyzdbnqlrzklh84egjziyfstlu4zdhk3yn6sxrwrsibuqhnxu6owh9y6aavpipoijyo0ytekaogobumb9nlwuloqsyy8mgfzhmjtz6mr4/dp0plrqjswfed//e4yi0dixz4jqe5nab1c1/swefxyur46qsgcl6dvbrcqwa1z+dp9wwcmivbkggh7tioieguaj/f8ezyu+plpkoqcu4wno3dk9t7zcbmni7n85alz4cgceqzutt2aznkzyxhxdzeea71lv+zqa/fvivj0qxep1ssoq72cbly0drqc+3waibcx+hdnptlhffwb+xjr307bpo8csm5z30cb2ilkrrc3smxqntneaufutf0q/0myqit9hn6muvnwo1ex6ijyq+/gbuu+5p</latexit> Recognition g <latexit sha1_base64="moeej9ypmg12hi+l6m/xxsminl4=">aaac5hicdzllahsxfibl6s2d3pj22y2okxrtmxmc7tkkm65ccnus8dhgi2tsyd2qzjhxhn4gu5bt8jb5ir5nzwco9bg9ipg5368j6ryvrnahwfarkzx6/otps63n6yuxr16/2d55e+x0bsnruy20ps2jy4ir1gcogp0ay4gsbtspz9+w/gtorona/ysfyunjjopxnbkiqcfk5asz5am84nf2n+tlq8cbim9efzvxnnrp3bdjtwvjffbbnbvkmyghjxy4fsykre2yixrgjmwqpsksuaff3tngjzezxpw2csnaq+zfozyrzi1kgz2swns12tl5lzaoofo69fyzgpiidwdvtccg8bibemwtoyawurbqebwrplnicyxypjqtfdunwkqixr7gah78mf+ch7aowjrg/vkb2or4bazlldpasxwbvmwqquclktb3efc7g4x+woc/nkmri1++lblhd9t4fhhpnff80qaxdbhsg1kdzigny8/bq94ux7u9povlp/a6+wfnb9hc79eh9anl6avar9/reeojijs6qbfolqmsq+q6uxmwjp1mzzu0fsntb6iq8ja=</latexit> N (f (~z ), <latexit sha1_base64="ufamb61cbqdk8cpcyuhfvjid7va=">aaacxnicdzhnahsxemfl7ve6/uqayy+iptblzw4itmeqxhikcdrowosertwbq9bhis06mulqr+g1eys+ud+msrou2m4hbh/m95+rrlpwujjmsl+d5nhjj0+fbt1px7x89frn9s7bgton5ddnrhp7utihumjoo0ajf7ufpkoj5+x0amhpz2cdmporzmsykxatrsu4w5gafdpg9pzqu5v1smxqtzg3okvaol3a6fwsxoy3cjryyzwb5lmni88sci4hpexjogz8yq5hgkvmctzil58b6ieygdpk2hg00mx27wrplhnzvuanyjhx62yr/bcbnlh9gxmh6wzb84elqkzsnhqxox0lcxzlpargryhvpxzclomyfyhncw033cjf9ngxqs+cv/sf8hbwado6eh9zokldvikizdrvxsgqzwiyycr4oor9fr783kajpxb8r+feqsgxk3im/ck6nt1gulzhopi2ybrsun18fzebyrdxy7nefrbfpths97xf3ph35cpjywdyqi7jkekttr6rh+so3cfhiu6a5obbmntaml2yesn33+gt4/a=</latexit> <latexit H <latexit sha1_base64="nieubnmdpyqx73brsmoxepsivxy=">aaac/xicdzfnbxmxeiad5assh03hymuiqmolihyrjjc4vhdhvbwjtjwyaer1vi0vf2hphgsz4rdw4iiqv34cvzjyb3c2k0q2mjklv/o8m7zncio4gyt51ykuxb5y9drw9fjgzvu3t7s7d46dli1la6qftqc5cuxwxqbaqbbtyxmruwan+ezfip/mmxvcq9ewngwkybnibaceqmrcffzmnjkp383mjoj3+d2uxwipz8qyqxfytnfdnmguadnd424v6sd14e2rnqkhmjga73q+zxnns8kuuegcg6ajgzenfjgvriqz0jfd6iycs2gqikjmrr7+zyufhmwef9qgowdx2b8rpjholwuenjla1lxzkvkvniyhedryxjksmkixfxwlwkdxamr4wi2jijzbegp5ecumu2ijhtdyom4ue0u1lernfmbvvpjn/lfavesarkm8p8tamypcgces5yx2bn2mfzukovjzefr7tpl7g43+woc/nkmra1/9lblhd9t4vgi0xmybzbowq+kw3bs9y01xvn9pk3766nhv4hmz5y10d91huyhft9abeomo0abr9bf9rz/qz+hd9cn6en29seadpuyuwovo229uzpfy</latexit> 1 <latexit sha1_base64="v9stdxa+kdraw2akggdjiw16qw0=">aaac43icdzllbhmxfiadkzcy3nqyzgmribehmokqwbkctvdvkuhbkukjj3omsekbbe9oavkj2fxdti/du/a2ookikqkcydkv8/0+9vfxqtmzlst+dzktbw8fpd5+kj599vzfy53dvroraknhqbvx5qwkfjitmhdmctjtbogoozyws89lfjohy5msx91cw0iqc8kqromlqw+hy1/okrv3w3inm/wyvebnkteii5o4hu92fhytrwsb0lforb3mmxyjt4xjlenii9qcjnrglmaypsqc7mivrhzw25iz4eqzuktdq+zfozwr1i5egz2cuklts2xyx2xyu+rjydopawes3h9u1rw7hzf94wkzqb1fregoyfgume6jidtfv0rtqsj3qoqgcuiljufbn/v3eqjrwbkdvd8vnnihhogdidjqzwhdpirmerv8ucx6pg++v1hoj8x913nkrotltinh/qin55erzohiyza4asbvg8wamatphhvehvkmoon38qyxf9nvhnxqpsa2eo3eohcorx/qatpex2iakjlobt2iuwssh8l1cnnvttrnnldolzlb396e8eg=</latexit> q (~z ~x) = N (g 1 (~x), H 2 (~x)) <latexit sha1_base64="gpycta341qfxpkhgbxclfwjisdc=">aaac7hicdzllbhmxfiad4vagwwjlnhyrulkqzvriskxgw6oqemkrxwnkctynfd+wpdmkxm/bdnvbhosn4g1w0heiezispv/n+31sn+ncc2zdlv3qjldu37l7b+9++udho8dpur2nj1zvhtahuvyzswjbypmkq8ccp2faucwktk+l+yc1p62psuzjz26p6vjgc8lkrrclqum392wc9izto3ofv0jul15nuv1skg0c7oq8ex3qxpgk1/mjpopugkpholz2lgfajt02jhfoq4oqszumc3xbr1fklkgd+83da3wzm1nykhoxdhct/xuhx8lapsiiu2a3s222tv6ljspxvht7jnxlqcq3b5uvh07bdspglblkhf9ggylh8a6qzldbxmv2psms9jioibccesrkhfy5f52hsa2c0ch7c+sudvei6aywvitlt21k0hkvg0dlro/z4a92cv2xup96joyifp1sgrk/aun6ewlncvy0waobqzayn2ae0jj2vd3kxxfymmizqf7ptf/wffmb9sbz8alsgxy8byfgizggq0dajbgc1+bhipnvyffk6saadjo9z8bwjne/atbt8wi=</latexit> 2 cov <latexit sha1_base64="eia10/qkrfpumu2uiu4ysfhot2y=">aaada3icdzjnixmxhmbt8w0dx7arry/binrby0wr9cisetntsoldxeh0hztntkf5i8mm7yycvfpfvmle14ufwm9j2g5ip/qhwmpze/l6z1gxamys/gpft27fuxtv73784ogjx/vtgydnrpyakwgwtoqlmtkeuuegllpglpqmii8zor/pp6z4euw0ovj8sktfrhxnbs0orjzyebv7dmynmnenucvujoap72bv4valpk6n/sy4znudpjesc+6ktbydundpftd6mu0kljkrfjnkzdbnlb05pc3fjpg4kw1rcm/rlaydfigtm3lrk3n4ijgtwegdhrbw7f49wyfuzjkpq5ijoznntjl/xyalld6ohbwqtetgzuzfyacvcpu+cei1wzytg0by03bwigdii2zdk8zxjshnldlhyuiykirvlt2r1pttylxyzl1mvioftobwi2gungq7jgwzsck7rajru9s7/s5cfyl2v5nqvsbxn8wiuzmmrhabvgtdrrnc1ecqcey1mps4td1tnnlxnpv7adjlp77uhl2vp8aeeaaegy5iwrtwbi7bkrgadl6ca3adfkrfom/r9+h6e41a9zynykuim9+cvftj</latexit> I) q (~z ~x) p(~z ~x, ) mean ~x 2

3 Variational EM: (in theory) E-step: train recognition networks g <latexit sha1_base64="moeej9ypmg12hi+l6m/xxsminl4=">aaac5hicdzllahsxfibl6s2d3pj22y2okxrtmxmc7tkkm65ccnus8dhgi2tsyd2qzjhxhn4gu5bt8jb5ir5nzwco9bg9ipg5368j6ryvrnahwfarkzx6/otps63n6yuxr16/2d55e+x0bsnruy20ps2jy4ir1gcogp0ay4gsbtspz9+w/gtorona/ysfyunjjopxnbkiqcfk5asz5am84nf2n+tlq8cbim9efzvxnnrp3bdjtwvjffbbnbvkmyghjxy4fsykre2yixrgjmwqpsksuaff3tngjzezxpw2csnaq+zfozyrzi1kgz2swns12tl5lzaoofo69fyzgpiidwdvtccg8bibemwtoyawurbqebwrplnicyxypjqtfdunwkqixr7gah78mf+ch7aowjrg/vkb2or4bazlldpasxwbvmwqquclktb3efc7g4x+woc/nkmri1++lblhd9t4fhhpnff80qaxdbhsg1kdzigny8/bq94ux7u9povlp/a6+wfnb9hc79eh9anl6avar9/reeojijs6qbfolqmsq+q6uxmwjp1mzzu0fsntb6iq8ja=</latexit> 1 and H <latexit sha1_base64="v9stdxa+kdraw2akggdjiw16qw0=">aaac43icdzllbhmxfiadkzcy3nqyzgmribehmokqwbkctvdvkuhbkukjj3omsekbbe9oavkj2fxdti/du/a2ookikqkcydkv8/0+9vfxqtmzlst+dzktbw8fpd5+kj599vzfy53dvroraknhqbvx5qwkfjitmhdmctjtbogoozyws89lfjohy5msx91cw0iqc8kqromlqw+hy1/okrv3w3inm/wyvebnkteii5o4hu92fhytrwsb0lforb3mmxyjt4xjlenii9qcjnrglmaypsqc7mivrhzw25iz4eqzuktdq+zfozwr1i5egz2cuklts2xyx2xyu+rjydopawes3h9u1rw7hzf94wkzqb1fregoyfgume6jidtfv0rtqsj3qoqgcuiljufbn/v3eqjrwbkdvd8vnnihhogdidjqzwhdpirmerv8ucx6pg++v1hoj8x913nkrotltinh/qin55erzohiyza4asbvg8wamatphhvehvkmoon38qyxf9nvhnxqpsa2eo3eohcorx/qatpex2iakjlobt2iuwssh8l1cnnvttrnnldolzlb396e8eg=</latexit> 2 to minimize KL q (~z ~x) p(~z ~x, ) <latexit sha1_base64="tkrsiidf3/gcl0kge9os5lblhc0=">aaaddnicdzllbhmxfiad4vbcpsks2vhesike0uxbgmufgyrqvstsvortyon4mly8tre9qxj33oetl8iodcgmpajvgycditkbi1n+fb7/+b4rzownw1+t4nr1gzdv7dxu37l77/5uz+/bszg5jnrijjf6nmagcibo0dll6ansfgcxpyfx/e3ftwqqdzpio10qos7wtlceewx9atj5/u49itkm9j5nkepzdxurea5rsej7tlxdcz94imxkle7dytufdlrhifwh3bzrlbqgjqpjxushmkqsz1rywrexoyhuduywtoxwwrzrbqjczi5ndoslwbk1y7c+xqmf+mwujll7jixcz/+ucdgzzpnf3plhm5omq5l/yqpcjq/gjgmvwyri1ujjzqgvsloqogwaesuxxmcimd8rjcnwmfh/oe02evqzkvmgxdqhjorsnblnuvluaqtv6dwzslkvfglonrzgsum3bvlqlcelq4mf30wl29+a6i/f/tdzqdppq5mszn1hexcltwtk4kijvjvyncg8bvoy7z89aj7ytjjeh0thiprwonvwuv4ao+areax6iaivwqf4c47aebdwfvyas/az+bj8c74hf1fwofxxpaqbevz+bu5d/8c=</latexit> M-step: train generative network f to maximize <latexit sha1_base64="z+vg35/w+jutrabd7h0ejs+88ma=">aaacyhicdzhnbhmxemed5assxy0cuvhesfyidiskofb0ungoiktasnk0cpzzxoq/zm+mjsxfeasu8as8ew+dk64qswaks3/n7z9jj2dipfbyfl862z279+4/2huyp3r85omz/ypn5940jkofg2ncymi8skghjwildkwdpiyslibz4xw/widzwugvulqwuuxki1pwhik1qmcvzgdzel9b9ip10f1rtqjl2jgbh3r+vlpdgwuauwted8vc4igwh4jlihnvelcmz9kvdjputiefhfwdi32dmlnag5eorrro/l0rmpj+qsbjqrjo/dzbjf/fhg3wh0zbansgah57ud1iioaupqdt4ycjxcbbubpprztpmgmc0x/leaxhmhulmj6gsuhfdjfhbrnjjkbnywixfrob0xuuhdpegg+bnqnhjusyqjr1d2umhzun/ljwv55tpxjftcqzdkfbehgt6ai4vdkg8xbmy562w27vclech/bkold+ftc9+tjuey+8jk/ig1ks9+sinjaz0iecspknfcc/sk+zza6z5a0167q1l8hgzf9/a5t+5qk=</latexit> Z <latexit sha1_base64="9kgqvtz7bzvm9xl/mryoalzynaq=">aaadbnicdzjnb9mwhmbd8dbcwwdhlhyv0izeluxicjzgwmkaet0m1v3loe5r1w/ytmjr5c6vl8in7trep+db4hyroin8purpnt/jt7+tac6ss5jfnejgzvu37+zcje/df/dwuxf38ylvpsf0qbrx5izdlnim6caxx+mznhsljnptbpzuxu8raixt8qnbadosecjzwqh2wrp3xyamhfw0rnrk9lc1vedvfb8irizqh+85vidb3yd5ei+7vasfratui7qrpddu8xi38xplipscskc4tnayjtqnpdaoeu7rgjwwakxmeekhquosqb359alq+dw4osyucu/y49r9e4thwtqfyejsyde1bbyy/8wgpsvejdytunrukuufipjdp+cqrtbnhhlhf0fgyljykyrtbdbxozfxjct9tjqqwoy+dk+q/bl/mdb1jnbg196fi6d0hzaazmbptbj0m6yknfki9qgi8/u09gdbe/2jup9mjowifhvsgrk/aunqhmhfczy3wbibyzaynwbwx+ha0/ylb4utg36a9nmpr3qhb5sfyac8bc/ahkjba3ai3onjmaaefawx4ar8il5e36lv0ev1noo0y56ajyqufgp/3/1o</latexit> q (~z ~x) log p(~x ~z )d~z 1 k j=1 <latexit sha1_base64="8fmbgnxeuw4kmg1ryiyqxujqqdi=">aaadexicdzllbhmxfiad4vbclyulg4siqsyizqok2cbvsgfvfym0leikchxp4o5vsj1deuonymulsenlwv6at8fjb0qsojklx+f77wof45hmzlo0/dlirl2/cfpwzu3mnbv37j9o7t48sao0hhaj4sqcjbclnenadcxxeqynxwle6emoelvkpxu1lin5wc017qs8ksxnbluygrzeiky1utoixg4w8vnwrydilmloz19nyvbaxnue6j1uzeanikrf8pwzhlbaasddbdwwws3aoi7j4w7joxorugoqhehy2l6watf32dhgoa1nvfqqmsnwhpailfhq2/erbwb4ngbgmfcmlungkvv3do+ftxmxik6b3drusmxyx6xxuvxv3zops0clusqulxw6bzfdgmnmkhf8hgumhsw7qjlfsveu9rtzrjj+jeoilmcemvkfp/dpsxdwgtm6ed9atukqs0bnslrawbpuu5joer48+j2i/a2d/ljcfz1hrks+fcnb3b9t4mowauujng2crq0wm6coqrgacezz5pc3xcl+j0s72fud9ugb+gpsgmfgcdgdgxgjdse7cay6giav4ajcgh/j5+rr8i25ulimjxrpi7awyeuvmwocja==</latexit> <latexit sha1_base64="0vsvqtfsztwhetwpfob2m5ypqtk=">aaadjhicdzllbhmxfiad4vbclyulg4siqswazuriskxga5uqskstfcer43gse99ke4ykjl+hlq/blh3qgk15fzx0vdujhmnsr//7pr6f46hmzlo0/v1lbt2+c/fe3v36g4ephj9p7d89taowhhai4sqcd7glnenaccxxeq4nxwli6dlw+n7fz0pqlfpys5tr2hn4lfnochbrgjq+iqy1utoixg4w8vnw0wcrlcqnp40sgwvcb4flepuzeajzaxit6vabkhedl6+wy3570gimrxrdcfdklwicqk4g+7ufakriiah0hgnru1mqxc9j4xjhnnrryangzirhtbulxilanl/foccx0rnbxjm4pinr9+yoj4w1czgmsyhdxg6zlfkv1i1c/rbnmdsfo5jchzqxhdofvw2ei2yocxwebsagxx+fzijjp1xsc72ojp1klbbyjjxisgy+7w+zedabmzp430do6rcpgm5gabwyddomjj3wphh0pymdd11h3h8zx0zevropwdwfb+nyfmljczxtg0uffttgwofpqmexz9td3hwn7vawtrjpr5th76ohsaeegxfgagtgdtgch8aj6aacvomlcan+jn+sn8mv5oiqmtsqpc/ariwxfwe68gra</latexit> k X 1 k X log p(~x ~zj ) ~zj q (~z ~x) <latexit sha1_base64="ece2xhfveaob+n5rgexjnffveok=">aaac9xicdzllbhmxgiwd4vags1pywcyiqiolopkkczyvbfhvrsjtpuw68jiexsq3bm80ibhei7bd3bbv0kfgbxdsesit+cvlr+c7/n0tfkpgjsmvtntr9p2797buxw8epnq83d15cmrkptezymmkpimqiywkmrdumnkineg8yos4mh5y8uoaaeol+gzniow4oho0pbjzyoxdz1m9yl/azfaov+azmtdd4hzl6tmrvntl+smq4kzig9edtr3mo53rbcxxxymwmcfjhmmi7mghbslmxmdzzyhceiroydbigtgxi7c6hicvgzogpdrhcatx7t8zholgzhkrkhzziwmzpfkvnqxs+w7kqfcvjqlflfrwdfojlzccx1qtbnk8ciq1dxufeii0wjbcwxxngpxjytksy5druxt36l6n3q8dq5v37jszuvmwblqacaokiesxkcield5lzejvuu/2nhr9idj/zg40d3x5uoyyo2jjehzottcctcgiays2mdzg6upw7gn7ktff0v4/tfrppze9/ffnb9gcz8elsats8bbsg4/geawabt/bbbgev9f59cp6gv3crknom+cpwkvo8jf7n/cw</latexit> 1 2 x 2 ~ f (~zj ) 2

4 <latexit sha1_base64="bkd/q8dcfg3nhrg+k+fr2nt58ki=">aaadlnicdzllahsxgixl6swpe3pazteijubqamzmodkuqrsjxyqu6irg2ymsa2w1gklimqltoxfkg3qd6kkkyz5gzxso9at9qxd4v6pbkcakm2pj+hsjunp33v2d3qfnh48ep3na2nt2zmshce0tyaw+ggndoro0b5nl9ejpivmxp+fjyw8rfl5sbzgun+1c0wgop4jljgabwmnriyqx6rf4dijom9s5th1sm5b2faev8wok2xrmd0yozrotl3jx8x6ikiodx8fpzu425rtvjrvxuubtkvsidao6tfca39bekiknwhkojrkksbjdh7vlhfpfriwhcpnlpkwdiaxoqrm69au93a+dccykdknyuo7+pcph3jhfpg7ohnuzqbnv819sunjscoiyuiwlgmw2ygoorysrbogeauosxwsbiwbhrjdmcmjhhpybtstovylzhiujq0yu3o3c68t7bwc18s6nkjuqjloprcbckgnotk0kouoz38q/ttafi/2v50tnga9usjb3j3vczgmtkyhzolhwyfkh4qnucaxjubs+gd4/qb/2bxhw6yzxn/n0pn30vvoju+afeak6iafvwre4bqegdwi4atfgbvymrqlr6ed0s7fgjwroc7bv0a/fkjwn0q==</latexit> <latexit sha1_base64="bkd/q8dcfg3nhrg+k+fr2nt58ki=">aaadlnicdzllahsxgixl6swpe3pazteijubqamzmodkuqrsjxyqu6irg2ymsa2w1gklimqltoxfkg3qd6kkkyz5gzxso9at9qxd4v6pbkcakm2pj+hsjunp33v2d3qfnh48ep3na2nt2zmshce0tyaw+ggndoro0b5nl9ejpivmxp+fjyw8rfl5sbzgun+1c0wgop4jljgabwmnriyqx6rf4dijom9s5th1sm5b2faev8wok2xrmd0yozrotl3jx8x6ikiodx8fpzu425rtvjrvxuubtkvsidao6tfca39bekiknwhkojrkksbjdh7vlhfpfriwhcpnlpkwdiaxoqrm69au93a+dccykdknyuo7+pcph3jhfpg7ohnuzqbnv819sunjscoiyuiwlgmw2ygoorysrbogeauosxwsbiwbhrjdmcmjhhpybtstovylzhiujq0yu3o3c68t7bwc18s6nkjuqjloprcbckgnotk0kouoz38q/ttafi/2v50tnga9usjb3j3vczgmtkyhzolhwyfkh4qnucaxjubs+gd4/qb/2bxhw6yzxn/n0pn30vvoju+afeak6iafvwre4bqegdwi4atfgbvymrqlr6ed0s7fgjwroc7bv0a/fkjwn0q==</latexit> <latexit sha1_base64="bkd/q8dcfg3nhrg+k+fr2nt58ki=">aaadlnicdzllahsxgixl6swpe3pazteijubqamzmodkuqrsjxyqu6irg2ymsa2w1gklimqltoxfkg3qd6kkkyz5gzxso9at9qxd4v6pbkcakm2pj+hsjunp33v2d3qfnh48ep3na2nt2zmshce0tyaw+ggndoro0b5nl9ejpivmxp+fjyw8rfl5sbzgun+1c0wgop4jljgabwmnriyqx6rf4dijom9s5th1sm5b2faev8wok2xrmd0yozrotl3jx8x6ikiodx8fpzu425rtvjrvxuubtkvsidao6tfca39bekiknwhkojrkksbjdh7vlhfpfriwhcpnlpkwdiaxoqrm69au93a+dccykdknyuo7+pcph3jhfpg7ohnuzqbnv819sunjscoiyuiwlgmw2ygoorysrbogeauosxwsbiwbhrjdmcmjhhpybtstovylzhiujq0yu3o3c68t7bwc18s6nkjuqjloprcbckgnotk0kouoz38q/ttafi/2v50tnga9usjb3j3vczgmtkyhzolhwyfkh4qnucaxjubs+gd4/qb/2bxhw6yzxn/n0pn30vvoju+afeak6iafvwre4bqegdwi4atfgbvymrqlr6ed0s7fgjwroc7bv0a/fkjwn0q==</latexit> <latexit sha1_base64="bkd/q8dcfg3nhrg+k+fr2nt58ki=">aaadlnicdzllahsxgixl6swpe3pazteijubqamzmodkuqrsjxyqu6irg2ymsa2w1gklimqltoxfkg3qd6kkkyz5gzxso9at9qxd4v6pbkcakm2pj+hsjunp33v2d3qfnh48ep3na2nt2zmshce0tyaw+ggndoro0b5nl9ejpivmxp+fjyw8rfl5sbzgun+1c0wgop4jljgabwmnriyqx6rf4dijom9s5th1sm5b2faev8wok2xrmd0yozrotl3jx8x6ikiodx8fpzu425rtvjrvxuubtkvsidao6tfca39bekiknwhkojrkksbjdh7vlhfpfriwhcpnlpkwdiaxoqrm69au93a+dccykdknyuo7+pcph3jhfpg7ohnuzqbnv819sunjscoiyuiwlgmw2ygoorysrbogeauosxwsbiwbhrjdmcmjhhpybtstovylzhiujq0yu3o3c68t7bwc18s6nkjuqjloprcbckgnotk0kouoz38q/ttafi/2v50tnga9usjb3j3vczgmtkyhzolhwyfkh4qnucaxjubs+gd4/qb/2bxhw6yzxn/n0pn30vvoju+afeak6iafvwre4bqegdwi4atfgbvymrqlr6ed0s7fgjwroc7bv0a/fkjwn0q==</latexit> <latexit sha1_base64="tg3nombrtlybqwjah5+nxyhg2qs=">aaadghicdzjnaxnbgmcn60trfev16guwcclo2a2cvqhflz2vcqytznjldjkbjjmxzwz2ttlmf/hqfxe8sk/15ldxkixien1g4c//95+359ms4mzyop7vig7dvnn3b/9e8/6dh48etw6enbtvakl7rhgllznskges9i2znf4wmmkrcxqrtd8v+uvftwfkfrtzgg4fhkuwm4jtsnlww1qt0k8qgsyguigzauyoq6dadmapq8wepynvogp2+bke1ezvbrz6tnwou/gq4k5iategdz2lb43vakrikai0hgnjbklc2khd2jlcqw+i0tackyke00gqegtqhm71tg9fbgcec6xdjy1cux+vcfgymxdzsapsj2ablc1/sufp86ohy7iolzvkfvbecmgvxdynjpimxpj5ejhofu4kyqshttnq2mytsfqzkcgwhdnezoxdlxuvel8jrc68c1fiqskhi6dvwjpcgbozu5joeo4dysp+lvgut7prn4j9bybml/dlswnm7nqbv7nak0rgbbssardybtmath0zjd3zhvkuoo91k7ibfhjdpn5x/wd74bl4djogaw/amtgbz6apcpgkrsen+bl9ib5fp6lrdtrq1guego2kbn4duxge1a==</latexit> <latexit sha1_base64="tg3nombrtlybqwjah5+nxyhg2qs=">aaadghicdzjnaxnbgmcn60trfev16guwcclo2a2cvqhflz2vcqytznjldjkbjjmxzwz2ttlmf/hqfxe8sk/15ldxkixien1g4c//95+359ms4mzyop7vig7dvnn3b/9e8/6dh48etw6enbtvakl7rhgllznskges9i2znf4wmmkrcxqrtd8v+uvftwfkfrtzgg4fhkuwm4jtsnlww1qt0k8qgsyguigzauyoq6dadmapq8wepynvogp2+bke1ezvbrz6tnwou/gq4k5iategdz2lb43vakrikai0hgnjbklc2khd2jlcqw+i0tackyke00gqegtqhm71tg9fbgcec6xdjy1cux+vcfgymxdzsapsj2ablc1/sufp86ohy7iolzvkfvbecmgvxdynjpimxpj5ejhofu4kyqshttnq2mytsfqzkcgwhdnezoxdlxuvel8jrc68c1fiqskhi6dvwjpcgbozu5joeo4dysp+lvgut7prn4j9bybml/dlswnm7nqbv7nak0rgbbssardybtmath0zjd3zhvkuoo91k7ibfhjdpn5x/wd74bl4djogaw/amtgbz6apcpgkrsen+bl9ib5fp6lrdtrq1guego2kbn4duxge1a==</latexit> <latexit sha1_base64="tg3nombrtlybqwjah5+nxyhg2qs=">aaadghicdzjnaxnbgmcn60trfev16guwcclo2a2cvqhflz2vcqytznjldjkbjjmxzwz2ttlmf/hqfxe8sk/15ldxkixien1g4c//95+359ms4mzyop7vig7dvnn3b/9e8/6dh48etw6enbtvakl7rhgllznskges9i2znf4wmmkrcxqrtd8v+uvftwfkfrtzgg4fhkuwm4jtsnlww1qt0k8qgsyguigzauyoq6dadmapq8wepynvogp2+bke1ezvbrz6tnwou/gq4k5iategdz2lb43vakrikai0hgnjbklc2khd2jlcqw+i0tackyke00gqegtqhm71tg9fbgcec6xdjy1cux+vcfgymxdzsapsj2ablc1/sufp86ohy7iolzvkfvbecmgvxdynjpimxpj5ejhofu4kyqshttnq2mytsfqzkcgwhdnezoxdlxuvel8jrc68c1fiqskhi6dvwjpcgbozu5joeo4dysp+lvgut7prn4j9bybml/dlswnm7nqbv7nak0rgbbssardybtmath0zjd3zhvkuoo91k7ibfhjdpn5x/wd74bl4djogaw/amtgbz6apcpgkrsen+bl9ib5fp6lrdtrq1guego2kbn4duxge1a==</latexit> <latexit sha1_base64="tg3nombrtlybqwjah5+nxyhg2qs=">aaadghicdzjnaxnbgmcn60trfev16guwcclo2a2cvqhflz2vcqytznjldjkbjjmxzwz2ttlmf/hqfxe8sk/15ldxkixien1g4c//95+359ms4mzyop7vig7dvnn3b/9e8/6dh48etw6enbtvakl7rhgllznskges9i2znf4wmmkrcxqrtd8v+uvftwfkfrtzgg4fhkuwm4jtsnlww1qt0k8qgsyguigzauyoq6dadmapq8wepynvogp2+bke1ezvbrz6tnwou/gq4k5iategdz2lb43vakrikai0hgnjbklc2khd2jlcqw+i0tackyke00gqegtqhm71tg9fbgcec6xdjy1cux+vcfgymxdzsapsj2ablc1/sufp86ohy7iolzvkfvbecmgvxdynjpimxpj5ejhofu4kyqshttnq2mytsfqzkcgwhdnezoxdlxuvel8jrc68c1fiqskhi6dvwjpcgbozu5joeo4dysp+lvgut7prn4j9bybml/dlswnm7nqbv7nak0rgbbssardybtmath0zjd3zhvkuoo91k7ibfhjdpn5x/wd74bl4djogaw/amtgbz6apcpgkrsen+bl9ib5fp6lrdtrq1guego2kbn4duxge1a==</latexit> <latexit sha1_base64="j/qkf+l/f5rhste2ophwsdl6qp8=">aaac+nicdzllbhmxfiad4vagwwosgzywevkrijqpkgbzwqy2vzfiwyloi49zpnhry8j2de1dv0x3qcx5df6at8fjr4gkccrlv873+xzbx0uluhvz9qut3lh56/adjbvpvfsphj7qbj7et7o2dazmc20oc2pbcaudx52aw8oalywag+l0w5wfngas1+qlm1uwkvry8ziz6mjq3h1kgqjs+aqtyyumu0zuza8+vrx3e1k/wwref3kreqinvffm5yezafzlui4jau0wzyo38tq4zgselnqwkspo6temo1rugh35xqucfheze1xqe5dyejh9e4en0tqzlkjtuje1q2ye/bcb1q58n/jcvbudxa4blbxatup5a+ajn8ccmevbmehxrjhnqahmxtdlu6lgk9nsujxxhksm+cp/og9hgthtbe+pinnvic2wm1tzsltytmkfu1egt8py3+fbb68v+mnx//xecuu+vymjwu+u4uys0gyyplsf5y04xwvx/i0iuxkhvujj/ppvaa+l/e1+nvxzz296o+/bn7cbnqhnaavl6c3aqr/rhhoghi7qjbpc35ol5dl5llxdw5nou+cjworkx28wnphp</latexit> <latexit sha1_base64="j/qkf+l/f5rhste2ophwsdl6qp8=">aaac+nicdzllbhmxfiad4vagwwosgzywevkrijqpkgbzwqy2vzfiwyloi49zpnhry8j2de1dv0x3qcx5df6at8fjr4gkccrlv873+xzbx0uluhvz9qut3lh56/adjbvpvfsphj7qbj7et7o2dazmc20oc2pbcaudx52aw8oalywag+l0w5wfngas1+qlm1uwkvry8ziz6mjq3h1kgqjs+aqtyyumu0zuza8+vrx3e1k/wwref3kreqinvffm5yezafzlui4jau0wzyo38tq4zgselnqwkspo6temo1rugh35xqucfheze1xqe5dyejh9e4en0tqzlkjtuje1q2ye/bcb1q58n/jcvbudxa4blbxatup5a+ajn8ccmevbmehxrjhnqahmxtdlu6lgk9nsujxxhksm+cp/og9hgthtbe+pinnvic2wm1tzsltytmkfu1egt8py3+fbb68v+mnx//xecuu+vymjwu+u4uys0gyyplsf5y04xwvx/i0iuxkhvujj/ppvaa+l/e1+nvxzz296o+/bn7cbnqhnaavl6c3aqr/rhhoghi7qjbpc35ol5dl5llxdw5nou+cjworkx28wnphp</latexit> <latexit sha1_base64="j/qkf+l/f5rhste2ophwsdl6qp8=">aaac+nicdzllbhmxfiad4vagwwosgzywevkrijqpkgbzwqy2vzfiwyloi49zpnhry8j2de1dv0x3qcx5df6at8fjr4gkccrlv873+xzbx0uluhvz9qut3lh56/adjbvpvfsphj7qbj7et7o2dazmc20oc2pbcaudx52aw8oalywag+l0w5wfngas1+qlm1uwkvry8ziz6mjq3h1kgqjs+aqtyyumu0zuza8+vrx3e1k/wwref3kreqinvffm5yezafzlui4jau0wzyo38tq4zgselnqwkspo6temo1rugh35xqucfheze1xqe5dyejh9e4en0tqzlkjtuje1q2ye/bcb1q58n/jcvbudxa4blbxatup5a+ajn8ccmevbmehxrjhnqahmxtdlu6lgk9nsujxxhksm+cp/og9hgthtbe+pinnvic2wm1tzsltytmkfu1egt8py3+fbb68v+mnx//xecuu+vymjwu+u4uys0gyyplsf5y04xwvx/i0iuxkhvujj/ppvaa+l/e1+nvxzz296o+/bn7cbnqhnaavl6c3aqr/rhhoghi7qjbpc35ol5dl5llxdw5nou+cjworkx28wnphp</latexit> <latexit sha1_base64="j/qkf+l/f5rhste2ophwsdl6qp8=">aaac+nicdzllbhmxfiad4vagwwosgzywevkrijqpkgbzwqy2vzfiwyloi49zpnhry8j2de1dv0x3qcx5df6at8fjr4gkccrlv873+xzbx0uluhvz9qut3lh56/adjbvpvfsphj7qbj7et7o2dazmc20oc2pbcaudx52aw8oalywag+l0w5wfngas1+qlm1uwkvry8ziz6mjq3h1kgqjs+aqtyyumu0zuza8+vrx3e1k/wwref3kreqinvffm5yezafzlui4jau0wzyo38tq4zgselnqwkspo6temo1rugh35xqucfheze1xqe5dyejh9e4en0tqzlkjtuje1q2ye/bcb1q58n/jcvbudxa4blbxatup5a+ajn8ccmevbmehxrjhnqahmxtdlu6lgk9nsujxxhksm+cp/og9hgthtbe+pinnvic2wm1tzsltytmkfu1egt8py3+fbb68v+mnx//xecuu+vymjwu+u4uys0gyyplsf5y04xwvx/i0iuxkhvujj/ppvaa+l/e1+nvxzz296o+/bn7cbnqhnaavl6c3aqr/rhhoghi7qjbpc35ol5dl5llxdw5nou+cjworkx28wnphp</latexit> M step is fine, but E-step is not. (Not obvious how to minimize KL term for phi1 and phi2) Alternate approach: use reparametrization trick and then do gradient descent on variational free energy Reparametrization trick: to get: ~z j q (~z ~x) z } { N (g 1 (~x),h 2 (~x)) do: ~ j N (0,I) ~z j =(H 2 (~x)) 1 2 ~ + g 1 (~x)

5 <latexit sha1_base64="ob4wpvzyhjwcazxivzumamj4diu=">aaadcnicdzllbhmxfiad4diyxjrcko1fbgoxrdmvet1uqmddkiosasnlkphjebkrvsn2dektv0gfpjtulix5ad4gjx0hmoejwfp1vt/h9jmekeantzjfreje/qcpd3yfxy+fph22195/fm5kqthpy8mkhkyqiywk0rfumjjqmia+yericvlxxs8qog2v4otdkjliaczoqtgyizw332rmzqa6gbzce5izkueonqr+3im1ykpftg/zdifpjuua2yktrqfuczbvt35mu4lltotfdbkztbnlrw5pszejps5kqxtcl2hghkekxikzufwdphwdmlnysb2wshcd/xuhq9yyjz8ej0d2bppslfwxg5a2ob45klrpicb3bxulg1bcvxfglgqclvsggbcm4a4qz5fg2iyexnemyfcsoudi6jiqku/g7m3q/sawwnnnxpmvyocjonvigcun2brjqeas8c4rqn2xene0veipxf7x09m88nvlmwku18tvitckylhogqsaxdubuazgwzcusehjmp+0oe1tcx7utznu+vld5/rd/rn2wuvwchyaflwhp+atoan9gme1uag34ht0hd1e36lbo2vuqve8absr/fgnorh+ug==</latexit> <latexit sha1_base64="ob4wpvzyhjwcazxivzumamj4diu=">aaadcnicdzllbhmxfiad4diyxjrcko1fbgoxrdmvet1uqmddkiosasnlkphjebkrvsn2dektv0gfpjtulix5ad4gjx0hmoejwfp1vt/h9jmekeantzjfreje/qcpd3yfxy+fph22195/fm5kqthpy8mkhkyqiywk0rfumjjqmia+yericvlxxs8qog2v4otdkjliaczoqtgyizw332rmzqa6gbzce5izkueonqr+3im1ykpftg/zdifpjuua2yktrqfuczbvt35mu4lltotfdbkztbnlrw5pszejps5kqxtcl2hghkekxikzufwdphwdmlnysb2wshcd/xuhq9yyjz8ej0d2bppslfwxg5a2ob45klrpicb3bxulg1bcvxfglgqclvsggbcm4a4qz5fg2iyexnemyfcsoudi6jiqku/g7m3q/sawwnnnxpmvyocjonvigcun2brjqeas8c4rqn2xene0veipxf7x09m88nvlmwku18tvitckylhogqsaxdubuazgwzcusehjmp+0oe1tcx7utznu+vld5/rd/rn2wuvwchyaflwhp+atoan9gme1uag34ht0hd1e36lbo2vuqve8absr/fgnorh+ug==</latexit> <latexit sha1_base64="ob4wpvzyhjwcazxivzumamj4diu=">aaadcnicdzllbhmxfiad4diyxjrcko1fbgoxrdmvet1uqmddkiosasnlkphjebkrvsn2dektv0gfpjtulix5ad4gjx0hmoejwfp1vt/h9jmekeantzjfreje/qcpd3yfxy+fph22195/fm5kqthpy8mkhkyqiywk0rfumjjqmia+yericvlxxs8qog2v4otdkjliaczoqtgyizw332rmzqa6gbzce5izkueonqr+3im1ykpftg/zdifpjuua2yktrqfuczbvt35mu4lltotfdbkztbnlrw5pszejps5kqxtcl2hghkekxikzufwdphwdmlnysb2wshcd/xuhq9yyjz8ej0d2bppslfwxg5a2ob45klrpicb3bxulg1bcvxfglgqclvsggbcm4a4qz5fg2iyexnemyfcsoudi6jiqku/g7m3q/sawwnnnxpmvyocjonvigcun2brjqeas8c4rqn2xene0veipxf7x09m88nvlmwku18tvitckylhogqsaxdubuazgwzcusehjmp+0oe1tcx7utznu+vld5/rd/rn2wuvwchyaflwhp+atoan9gme1uag34ht0hd1e36lbo2vuqve8absr/fgnorh+ug==</latexit> <latexit sha1_base64="ob4wpvzyhjwcazxivzumamj4diu=">aaadcnicdzllbhmxfiad4diyxjrcko1fbgoxrdmvet1uqmddkiosasnlkphjebkrvsn2dektv0gfpjtulix5ad4gjx0hmoejwfp1vt/h9jmekeantzjfreje/qcpd3yfxy+fph22195/fm5kqthpy8mkhkyqiywk0rfumjjqmia+yericvlxxs8qog2v4otdkjliaczoqtgyizw332rmzqa6gbzce5izkueonqr+3im1ykpftg/zdifpjuua2yktrqfuczbvt35mu4lltotfdbkztbnlrw5pszejps5kqxtcl2hghkekxikzufwdphwdmlnysb2wshcd/xuhq9yyjz8ej0d2bppslfwxg5a2ob45klrpicb3bxulg1bcvxfglgqclvsggbcm4a4qz5fg2iyexnemyfcsoudi6jiqku/g7m3q/sawwnnnxpmvyocjonvigcun2brjqeas8c4rqn2xene0veipxf7x09m88nvlmwku18tvitckylhogqsaxdubuazgwzcusehjmp+0oe1tcx7utznu+vld5/rd/rn2wuvwchyaflwhp+atoan9gme1uag34ht0hd1e36lbo2vuqve8absr/fgnorh+ug==</latexit> <latexit sha1_base64="2njg/mbbrzx4t9qq9cx6zkyrv9m=">aaadenicdzllbhoxgiu90etklyhtshslvalegs1ekdpnpkjddivsqssrmiym8yavj+3ahgo4fo0+vv+g75jfzycqaukvjxz0f8fxmypfmbfx/duqvb89fvl04fnt+yuxrw7rr6+vjcw0ot0iudq3i2woz4l2lloc3ihnct7i9hp0+3nfr2dugybfn7tqdjdjiwazi9igvlr/eq6rkflusfped7sqmwhh9xspkwuh2tjld2g2t1kbik4z2wqdnedvkpt3peey2sm1ua07awjx1j5emcbelezga3v3wlieataz2gfcj204lilm6834jc4l7otki5pgu5fpufqljsupcios4diyfhiro3byw0y49tvugkowucut2g9s4jyagssf0mn3otogmdthc/cvu//ocdg3zpgpgjphdmp22ar5eosxnvs4ceyowljb1htlbydwwluacmw0jzyvgsbes3bwsky4vjsnmdvqsnafroy5fmmxkpl5n3tve++3gdxkozdevioftobwy2gunhtbjgwd8sy7ds6jd6d7c/212p96ujopfhvtgrnr7ulzpnazjxc+c5ybsnwfvjknqkeylowvhfyt3bt3xdxpsrkfjf/pmhefnn/caxglgqafevabxiav4bl0aah3uspqrmev+2qj2q521tzktjnzbmxv9ewp8uyldw==</latexit> <latexit sha1_base64="2njg/mbbrzx4t9qq9cx6zkyrv9m=">aaadenicdzllbhoxgiu90etklyhtshslvalegs1ekdpnpkjddivsqssrmiym8yavj+3ahgo4fo0+vv+g75jfzycqaukvjxz0f8fxmypfmbfx/duqvb89fvl04fnt+yuxrw7rr6+vjcw0ot0iudq3i2woz4l2lloc3ihnct7i9hp0+3nfr2dugybfn7tqdjdjiwazi9igvlr/eq6rkflusfped7sqmwhh9xspkwuh2tjld2g2t1kbik4z2wqdnedvkpt3peey2sm1ua07awjx1j5emcbelezga3v3wlieataz2gfcj204lilm6834jc4l7otki5pgu5fpufqljsupcios4diyfhiro3byw0y49tvugkowucut2g9s4jyagssf0mn3otogmdthc/cvu//ocdg3zpgpgjphdmp22ar5eosxnvs4ceyowljb1htlbydwwluacmw0jzyvgsbes3bwsky4vjsnmdvqsnafroy5fmmxkpl5n3tve++3gdxkozdevioftobwy2gunhtbjgwd8sy7ds6jd6d7c/212p96ujopfhvtgrnr7ulzpnazjxc+c5ybsnwfvjknqkeylowvhfyt3bt3xdxpsrkfjf/pmhefnn/caxglgqafevabxiav4bl0aah3uspqrmev+2qj2q521tzktjnzbmxv9ewp8uyldw==</latexit> <latexit sha1_base64="2njg/mbbrzx4t9qq9cx6zkyrv9m=">aaadenicdzllbhoxgiu90etklyhtshslvalegs1ekdpnpkjddivsqssrmiym8yavj+3ahgo4fo0+vv+g75jfzycqaukvjxz0f8fxmypfmbfx/duqvb89fvl04fnt+yuxrw7rr6+vjcw0ot0iudq3i2woz4l2lloc3ihnct7i9hp0+3nfr2dugybfn7tqdjdjiwazi9igvlr/eq6rkflusfped7sqmwhh9xspkwuh2tjld2g2t1kbik4z2wqdnedvkpt3peey2sm1ua07awjx1j5emcbelezga3v3wlieataz2gfcj204lilm6834jc4l7otki5pgu5fpufqljsupcios4diyfhiro3byw0y49tvugkowucut2g9s4jyagssf0mn3otogmdthc/cvu//ocdg3zpgpgjphdmp22ar5eosxnvs4ceyowljb1htlbydwwluacmw0jzyvgsbes3bwsky4vjsnmdvqsnafroy5fmmxkpl5n3tve++3gdxkozdevioftobwy2gunhtbjgwd8sy7ds6jd6d7c/212p96ujopfhvtgrnr7ulzpnazjxc+c5ybsnwfvjknqkeylowvhfyt3bt3xdxpsrkfjf/pmhefnn/caxglgqafevabxiav4bl0aah3uspqrmev+2qj2q521tzktjnzbmxv9ewp8uyldw==</latexit> <latexit sha1_base64="2njg/mbbrzx4t9qq9cx6zkyrv9m=">aaadenicdzllbhoxgiu90etklyhtshslvalegs1ekdpnpkjddivsqssrmiym8yavj+3ahgo4fo0+vv+g75jfzycqaukvjxz0f8fxmypfmbfx/duqvb89fvl04fnt+yuxrw7rr6+vjcw0ot0iudq3i2woz4l2lloc3ihnct7i9hp0+3nfr2dugybfn7tqdjdjiwazi9igvlr/eq6rkflusfped7sqmwhh9xspkwuh2tjld2g2t1kbik4z2wqdnedvkpt3peey2sm1ua07awjx1j5emcbelezga3v3wlieataz2gfcj204lilm6834jc4l7otki5pgu5fpufqljsupcios4diyfhiro3byw0y49tvugkowucut2g9s4jyagssf0mn3otogmdthc/cvu//ocdg3zpgpgjphdmp22ar5eosxnvs4ceyowljb1htlbydwwluacmw0jzyvgsbes3bwsky4vjsnmdvqsnafroy5fmmxkpl5n3tve++3gdxkozdevioftobwy2gunhtbjgwd8sy7ds6jd6d7c/212p96ujopfhvtgrnr7ulzpnazjxc+c5ybsnwfvjknqkeylowvhfyt3bt3xdxpsrkfjf/pmhefnn/caxglgqafevabxiav4bl0aah3uspqrmev+2qj2q521tzktjnzbmxv9ewp8uyldw==</latexit> <latexit sha1_base64="9+yq9slq2hsor1wrtp14trmz9t4=">aaadw3icdzlfbtmwfmadhj8jdncbuolgoklqjjiscqlukca44aoaet0m1w3kuk5jzbez28nagr8hl8df3gw3jrbtx5eiffp+34njczipodmmjn8hrfdbw0epd55et9uhz553jl5cafkpqgdecqmujlhtzgqdggy4vsovxcwe08vj9dcvv6yp0kykh2zr0lgbz4jljgdjrbrzi2b0bijdfallnxm37kpehie3kspz1kp1mmu/ut1p2thicgyrp5kzqpqptcwse2v0romznbp87ow9zq4pnsvnce7xlkw73fgkxhfcf0kjuqcp8/qo+iwmklqffyzwrpuwiuszslgzrjh1eao0ltg5xjm69flgguqrxu/iwbfemcjmkv/4263dfzsslrrefboflldj9s5bmfexywwytyplrfkzksjmokzi0ei4gjecmkwj4qsvmfhmfyskofbdm34puyqevswyklcywj/32tmxfz84tw2mkp21y2rk6fwr0cgsdck13y5jqxoeobtztuls6d6l/kbmfzn9vxi+uinb3pz3ct33tkyeznfbsghlxubl3sdkjensumjvp9nd9r64od1j4ppk+4fu2zfmtzgar8eb0amj+ajowddwdgaaglsgdnrbyesudmmobg+iradpeqm2knz1bz4igsu=</latexit> <latexit sha1_base64="9+yq9slq2hsor1wrtp14trmz9t4=">aaadw3icdzlfbtmwfmadhj8jdncbuolgoklqjjiscqlukca44aoaet0m1w3kuk5jzbez28nagr8hl8df3gw3jrbtx5eiffp+34njczipodmmjn8hrfdbw0epd55et9uhz553jl5cafkpqgdecqmujlhtzgqdggy4vsovxcwe08vj9dcvv6yp0kykh2zr0lgbz4jljgdjrbrzi2b0bijdfallnxm37kpehie3kspz1kp1mmu/ut1p2thicgyrp5kzqpqptcwse2v0romznbp87ow9zq4pnsvnce7xlkw73fgkxhfcf0kjuqcp8/qo+iwmklqffyzwrpuwiuszslgzrjh1eao0ltg5xjm69flgguqrxu/iwbfemcjmkv/4263dfzsslrrefboflldj9s5bmfexywwytyplrfkzksjmokzi0ei4gjecmkwj4qsvmfhmfyskofbdm34puyqevswyklcywj/32tmxfz84tw2mkp21y2rk6fwr0cgsdck13y5jqxoeobtztuls6d6l/kbmfzn9vxi+uinb3pz3ct33tkyeznfbsghlxubl3sdkjensumjvp9nd9r64od1j4ppk+4fu2zfmtzgar8eb0amj+ajowddwdgaaglsgdnrbyesudmmobg+iradpeqm2knz1bz4igsu=</latexit> <latexit sha1_base64="9+yq9slq2hsor1wrtp14trmz9t4=">aaadw3icdzlfbtmwfmadhj8jdncbuolgoklqjjiscqlukca44aoaet0m1w3kuk5jzbez28nagr8hl8df3gw3jrbtx5eiffp+34njczipodmmjn8hrfdbw0epd55et9uhz553jl5cafkpqgdecqmujlhtzgqdggy4vsovxcwe08vj9dcvv6yp0kykh2zr0lgbz4jljgdjrbrzi2b0bijdfallnxm37kpehie3kspz1kp1mmu/ut1p2thicgyrp5kzqpqptcwse2v0romznbp87ow9zq4pnsvnce7xlkw73fgkxhfcf0kjuqcp8/qo+iwmklqffyzwrpuwiuszslgzrjh1eao0ltg5xjm69flgguqrxu/iwbfemcjmkv/4263dfzsslrrefboflldj9s5bmfexywwytyplrfkzksjmokzi0ei4gjecmkwj4qsvmfhmfyskofbdm34puyqevswyklcywj/32tmxfz84tw2mkp21y2rk6fwr0cgsdck13y5jqxoeobtztuls6d6l/kbmfzn9vxi+uinb3pz3ct33tkyeznfbsghlxubl3sdkjensumjvp9nd9r64od1j4ppk+4fu2zfmtzgar8eb0amj+ajowddwdgaaglsgdnrbyesudmmobg+iradpeqm2knz1bz4igsu=</latexit> <latexit sha1_base64="9+yq9slq2hsor1wrtp14trmz9t4=">aaadw3icdzlfbtmwfmadhj8jdncbuolgoklqjjiscqlukca44aoaet0m1w3kuk5jzbez28nagr8hl8df3gw3jrbtx5eiffp+34njczipodmmjn8hrfdbw0epd55et9uhz553jl5cafkpqgdecqmujlhtzgqdggy4vsovxcwe08vj9dcvv6yp0kykh2zr0lgbz4jljgdjrbrzi2b0bijdfallnxm37kpehie3kspz1kp1mmu/ut1p2thicgyrp5kzqpqptcwse2v0romznbp87ow9zq4pnsvnce7xlkw73fgkxhfcf0kjuqcp8/qo+iwmklqffyzwrpuwiuszslgzrjh1eao0ltg5xjm69flgguqrxu/iwbfemcjmkv/4263dfzsslrrefboflldj9s5bmfexywwytyplrfkzksjmokzi0ei4gjecmkwj4qsvmfhmfyskofbdm34puyqevswyklcywj/32tmxfz84tw2mkp21y2rk6fwr0cgsdck13y5jqxoeobtztuls6d6l/kbmfzn9vxi+uinb3pz3ct33tkyeznfbsghlxubl3sdkjensumjvp9nd9r64od1j4ppk+4fu2zfmtzgar8eb0amj+ajowddwdgaaglsgdnrbyesudmmobg+iradpeqm2knz1bz4igsu=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> log p(x) = = NX i=1 NX i=1 NX i=1 NX i=1 log p(~x i ) Z Z 1 k q (~z i ~x i ) log q (~z i ~x i ) kx j=1 VAE objective apple p(~xi, ~z i ) q (~z i ~x i ) d~z i log p(~x i ~z i, ) + log log p(~x i ~z ij, ) apple p(~zi ) q (~z i ~x i ) d~z i ) 1 KL (q (~z i ~x i ) p(~z i A can use analytic formula for KL of 2 Gaussians N (g 1 (~x),h 2 (~x)) N (0,I)

6 <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="dvlpku8ykz0bbyg0bgnimrkjgq4=">aaac/hicdzllbhmxfiad4vagwwobjdywevk7ijqpkgcdvmggvvuk0lak08hxpi1v37a9qxljpk13qcx5cl6at8gtjhczwjes/t7fbx8f2xpnmxvz9qut3lh56/adrbvpvfsphj7qbj8+sqo0ha6i4sqctlclnek6cmxxeqinxwlc6fhk/h3njytqlfpyk1toohl4tlkceexiatx9+nmm9iztogr5fvxzxfgw6nqyo+72sn62crgp8kb0qboh4+3otzrvpbruoskxtcm8027ksxgmcbpsvfqqmtnhz3qypcsc2pfftrdgi5izwkkzoksdq+zfkzww1i7ejdofdjpbznxyx2xyuulnydops0clus5ulbw6bev7gfnmkhf8equmhswzqjldbhmxby1nkarfibicy6lhtfbbn/qxeqjrwbkdvd9ftukqs0bnslrawbpuu5loebe8kul+pg9+b2ojpxb3x8+bezhxnrlm/uebv/nik0rgva2wdvi2adw2qshkxpumaxz/vp3am+jor59n/fzjq97+u+ynbifn4dnyatl4dfbbb3aibocac3ajrsd35ck5tl4lv9fwpnoseqlwivnxgyiy+za=</latexit> <latexit sha1_base64="dvlpku8ykz0bbyg0bgnimrkjgq4=">aaac/hicdzllbhmxfiad4vagwwobjdywevk7ijqpkgcdvmggvvuk0lak08hxpi1v37a9qxljpk13qcx5cl6at8gtjhczwjes/t7fbx8f2xpnmxvz9qut3lh56/adrbvpvfsphj7qbj8+sqo0ha6i4sqctlclnek6cmxxeqinxwlc6fhk/h3njytqlfpyk1toohl4tlkceexiatx9+nmm9iztogr5fvxzxfgw6nqyo+72sn62crgp8kb0qboh4+3otzrvpbruoskxtcm8027ksxgmcbpsvfqqmtnhz3qypcsc2pfftrdgi5izwkkzoksdq+zfkzww1i7ejdofdjpbznxyx2xyuulnydops0clus5ulbw6bev7gfnmkhf8equmhswzqjldbhmxby1nkarfibicy6lhtfbbn/qxeqjrwbkdvd9ftukqs0bnslrawbpuu5loebe8kul+pg9+b2ojpxb3x8+bezhxnrlm/uebv/nik0rgva2wdvi2adw2qshkxpumaxz/vp3am+jor59n/fzjq97+u+ynbifn4dnyatl4dfbbb3aibocac3ajrsd35ck5tl4lv9fwpnoseqlwivnxgyiy+za=</latexit> <latexit sha1_base64="dvlpku8ykz0bbyg0bgnimrkjgq4=">aaac/hicdzllbhmxfiad4vagwwobjdywevk7ijqpkgcdvmggvvuk0lak08hxpi1v37a9qxljpk13qcx5cl6at8gtjhczwjes/t7fbx8f2xpnmxvz9qut3lh56/adrbvpvfsphj7qbj8+sqo0ha6i4sqctlclnek6cmxxeqinxwlc6fhk/h3njytqlfpyk1toohl4tlkceexiatx9+nmm9iztogr5fvxzxfgw6nqyo+72sn62crgp8kb0qboh4+3otzrvpbruoskxtcm8027ksxgmcbpsvfqqmtnhz3qypcsc2pfftrdgi5izwkkzoksdq+zfkzww1i7ejdofdjpbznxyx2xyuulnydops0clus5ulbw6bev7gfnmkhf8equmhswzqjldbhmxby1nkarfibicy6lhtfbbn/qxeqjrwbkdvd9ftukqs0bnslrawbpuu5loebe8kul+pg9+b2ojpxb3x8+bezhxnrlm/uebv/nik0rgva2wdvi2adw2qshkxpumaxz/vp3am+jor59n/fzjq97+u+ynbifn4dnyatl4dfbbb3aibocac3ajrsd35ck5tl4lv9fwpnoseqlwivnxgyiy+za=</latexit> <latexit sha1_base64="dvlpku8ykz0bbyg0bgnimrkjgq4=">aaac/hicdzllbhmxfiad4vagwwobjdywevk7ijqpkgcdvmggvvuk0lak08hxpi1v37a9qxljpk13qcx5cl6at8gtjhczwjes/t7fbx8f2xpnmxvz9qut3lh56/adrbvpvfsphj7qbj8+sqo0ha6i4sqctlclnek6cmxxeqinxwlc6fhk/h3njytqlfpyk1toohl4tlkceexiatx9+nmm9iztogr5fvxzxfgw6nqyo+72sn62crgp8kb0qboh4+3otzrvpbruoskxtcm8027ksxgmcbpsvfqqmtnhz3qypcsc2pfftrdgi5izwkkzoksdq+zfkzww1i7ejdofdjpbznxyx2xyuulnydops0clus5ulbw6bev7gfnmkhf8equmhswzqjldbhmxby1nkarfibicy6lhtfbbn/qxeqjrwbkdvd9ftukqs0bnslrawbpuu5loebe8kul+pg9+b2ojpxb3x8+bezhxnrlm/uebv/nik0rgva2wdvi2adw2qshkxpumaxz/vp3am+jor59n/fzjq97+u+ynbifn4dnyatl4dfbbb3aibocac3ajrsd35ck5tl4lv9fwpnoseqlwivnxgyiy+za=</latexit> NX 1 k kx VAE objective log p(~x i ~z ij, ) ) 1 KL (q (~z i ~x i ) p(~z i A i=1 j=1 N (g 1 (~x),h 2 (~x)) N (0,I) NOTE: this does not mean that VAE is trying to make q (~z ~x) =p(~z)

7 <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="wx3fobtig5stux3z4lbuxlph3vk=">aaadbxicdzltalmxgmft1zdz3zpln5qzlgilrpwzbp0ijplfumoeuw2atqrptps1j4ljtm2b5gk8n2/aq9gtmnmdhm31gqn//r//k7fndbrnxkbr79jo+dbto3d371xup3j46hf178mzkzkmte0kl/piga3ltnc2zzbtc6uptgecng8mh3n+pqxamcm+2bmi3rspbesywtzy/epphjxscgyhmlnad+xd7hstidhnbb0lghmxezfxbb7m8srgoykqjqazplui6sl0xfo3yi6pxq14cd9/qsdsvk9/7ym1zvu8kedcuggxy1vncbp5qagrzqoxbfsrtagzrqpui7gqnvduax+v9asnjclskizh2jhohcnbdvhbrjj1fzqzqjcz4bhtbclwsk3xrd7mw1fbgcje6vajc1fu3x0op8bm00fiptiozsblzx+xtmat913hhmosfer6oytj0eqydwaomabe8nkqmggwzgrjgiehtmfmlqos9aerayrf0cempt713ghs/tqwwnnneshk5cmw0gosjjkgrsekogoeehczykothw4i9r+zlk4dz29kmhetttydbtqlbm42waiai01alskqcpjxkxwlzd/enpa2odtqxlez/vq2dnxs/am74bl4ceogbu/amfgetkebehbv2i8dlf7sxjx3y8/lb9frnvlr8xssvfn1hwpmh90=</latexit> <latexit sha1_base64="obuwvjnr4yfjbmwk5u7tlcxh820=">aaadcnicdzldbtmwfmedly8rvjq4gzegmqktw5vushcdnmhnrqyh0w1s3uao67smsr3ftmjn+hv4ml6a99gd4hrhwjm4uqs/zu9/tpjztpyltokg+o1tte7df/bw+5h/+mntz8/boy/olcxyqgdepjk/ilgikrn0ojlo6uwwu8zjlj7h8y8vpy9prpgu3/qyoyoop4iljgdtulh75yciku5ytexozdxcpaoegfbd3kr3kwjtjsd9u1qhchexeaitcokz1rjfbno9jgzkzizq2+6a748nuim2rlvjm3xd9r2c1ubwr7lqsb9ajfu+h7u7qs9yb7wrwlp0qb2n0y73c00kktgvmqryqweyzhpkck4zsan1uafohskct+nqsye5vsozhpyfey4zgynm3sm0xgdvvxjmlvry2dk51jpvzfxyx2xy6otjydcrfzokcv2ipeihlrdaapywnbkdlp3ajgfuwygzytcx7xbl+0jqh0ryjsxeiczka8bmmlr2e+g8s8amkzazm/ee1dkwkpokbtqkolm0srsbats6sej/ek5y7nj1pwsn5qsjy4wjjsvw0qsxnbhsancsnujoslqkw+0/bg77rjjr98kgf3593zn6xf/cnngf3oaucmehcasowskyaakuvf2v473dumrttl636rpz8uqal2ajwgd/aawtitk=</latexit> <latexit sha1_base64="obuwvjnr4yfjbmwk5u7tlcxh820=">aaadcnicdzldbtmwfmedly8rvjq4gzegmqktw5vushcdnmhnrqyh0w1s3uao67smsr3ftmjn+hv4ml6a99gd4hrhwjm4uqs/zu9/tpjztpyltokg+o1tte7df/bw+5h/+mntz8/boy/olcxyqgdepjk/ilgikrn0ojlo6uwwu8zjlj7h8y8vpy9prpgu3/qyoyoop4iljgdtulh75yciku5ytexozdxcpaoegfbd3kr3kwjtjsd9u1qhchexeaitcokz1rjfbno9jgzkzizq2+6a748nuim2rlvjm3xd9r2c1ubwr7lqsb9ajfu+h7u7qs9yb7wrwlp0qb2n0y73c00kktgvmqryqweyzhpkck4zsan1uafohskct+nqsye5vsozhpyfey4zgynm3sm0xgdvvxjmlvry2dk51jpvzfxyx2xy6otjydcrfzokcv2ipeihlrdaapywnbkdlp3ajgfuwygzytcx7xbl+0jqh0ryjsxeiczka8bmmlr2e+g8s8amkzazm/ee1dkwkpokbtqkolm0srsbats6sej/ek5y7nj1pwsn5qsjy4wjjsvw0qsxnbhsancsnujoslqkw+0/bg77rjjr98kgf3593zn6xf/cnngf3oaucmehcasowskyaakuvf2v473dumrttl636rpz8uqal2ajwgd/aawtitk=</latexit> <latexit sha1_base64="obuwvjnr4yfjbmwk5u7tlcxh820=">aaadcnicdzldbtmwfmedly8rvjq4gzegmqktw5vushcdnmhnrqyh0w1s3uao67smsr3ftmjn+hv4ml6a99gd4hrhwjm4uqs/zu9/tpjztpyltokg+o1tte7df/bw+5h/+mntz8/boy/olcxyqgdepjk/ilgikrn0ojlo6uwwu8zjlj7h8y8vpy9prpgu3/qyoyoop4iljgdtulh75yciku5ytexozdxcpaoegfbd3kr3kwjtjsd9u1qhchexeaitcokz1rjfbno9jgzkzizq2+6a748nuim2rlvjm3xd9r2c1ubwr7lqsb9ajfu+h7u7qs9yb7wrwlp0qb2n0y73c00kktgvmqryqweyzhpkck4zsan1uafohskct+nqsye5vsozhpyfey4zgynm3sm0xgdvvxjmlvry2dk51jpvzfxyx2xy6otjydcrfzokcv2ipeihlrdaapywnbkdlp3ajgfuwygzytcx7xbl+0jqh0ryjsxeiczka8bmmlr2e+g8s8amkzazm/ee1dkwkpokbtqkolm0srsbats6sej/ek5y7nj1pwsn5qsjy4wjjsvw0qsxnbhsancsnujoslqkw+0/bg77rjjr98kgf3593zn6xf/cnngf3oaucmehcasowskyaakuvf2v473dumrttl636rpz8uqal2ajwgd/aawtitk=</latexit> <latexit sha1_base64="obuwvjnr4yfjbmwk5u7tlcxh820=">aaadcnicdzldbtmwfmedly8rvjq4gzegmqktw5vushcdnmhnrqyh0w1s3uao67smsr3ftmjn+hv4ml6a99gd4hrhwjm4uqs/zu9/tpjztpyltokg+o1tte7df/bw+5h/+mntz8/boy/olcxyqgdepjk/ilgikrn0ojlo6uwwu8zjlj7h8y8vpy9prpgu3/qyoyoop4iljgdtulh75yciku5ytexozdxcpaoegfbd3kr3kwjtjsd9u1qhchexeaitcokz1rjfbno9jgzkzizq2+6a748nuim2rlvjm3xd9r2c1ubwr7lqsb9ajfu+h7u7qs9yb7wrwlp0qb2n0y73c00kktgvmqryqweyzhpkck4zsan1uafohskct+nqsye5vsozhpyfey4zgynm3sm0xgdvvxjmlvry2dk51jpvzfxyx2xy6otjydcrfzokcv2ipeihlrdaapywnbkdlp3ajgfuwygzytcx7xbl+0jqh0ryjsxeiczka8bmmlr2e+g8s8amkzazm/ee1dkwkpokbtqkolm0srsbats6sej/ek5y7nj1pwsn5qsjy4wjjsvw0qsxnbhsancsnujoslqkw+0/bg77rjjr98kgf3593zn6xf/cnngf3oaucmehcasowskyaakuvf2v473dumrttl636rpz8uqal2ajwgd/aawtitk=</latexit> <latexit sha1_base64="+cjv0oxg1vysmlcu4bihsyhivw8=">aaadkxicdzllbhmxfiad4vbclyulg6srulk0momqyinuwyzvvctsvoqtken4ehnfrrznsol4idjybjwbo+i2peiddfq1crzj0q//+z0en+nhzpmxcfynft25e+/+g72h9uepnzx91th/fmpuoqnteswvph9iqzmttguz5fq81xsliadnw+nhft8rqtzmys92nto+wgpjmkawdvba6lyhenlmy+is76yeiloi1lgv/pbdroanbr60/xkjylnk4bhmumqn1ojdvc7w+dfl5acdnppxk14x3bvjjzqgqpn0v/ytjrqpbjwwcgxml4lz23dyw0y49xvugjpjmsvj2gtsykfn360v7ugr4ixgpnry0sk1e3uhw8kyurigpmb2yrbzyvwx6xu2e9d3toafpzjch5qvhfofv12ei6ypsxwebcaahx+fzijdu2zodb2ojp1glbbyjhxisvru4i4s7zeb1bl3bocsyn04alqnpcmvozsxjemez96hm3hsfogmyv+b6wgr+oqmbhpx2cbllncsejjbboskllbbtajtxw9jt7ahvcto260kbiwf3zspp1qpya+8bafgectgltggn8aj6aicfoalcan+rt+jx9hv6oi6gtwqps/arkwxv6gmdkw=</latexit> <latexit sha1_base64="+cjv0oxg1vysmlcu4bihsyhivw8=">aaadkxicdzllbhmxfiad4vbclyulg6srulk0momqyinuwyzvvctsvoqtken4ehnfrrznsol4idjybjwbo+i2peiddfq1crzj0q//+z0en+nhzpmxcfynft25e+/+g72h9uepnzx91th/fmpuoqnteswvph9iqzmttguz5fq81xsliadnw+nhft8rqtzmys92nto+wgpjmkawdvba6lyhenlmy+is76yeiloi1lgv/pbdroanbr60/xkjylnk4bhmumqn1ojdvc7w+dfl5acdnppxk14x3bvjjzqgqpn0v/ytjrqpbjwwcgxml4lz23dyw0y49xvugjpjmsvj2gtsykfn360v7ugr4ixgpnry0sk1e3uhw8kyurigpmb2yrbzyvwx6xu2e9d3toafpzjch5qvhfofv12ei6ypsxwebcaahx+fzijdu2zodb2ojp1glbbyjhxisvru4i4s7zeb1bl3bocsyn04alqnpcmvozsxjemez96hm3hsfogmyv+b6wgr+oqmbhpx2cbllncsejjbboskllbbtajtxw9jt7ahvcto260kbiwf3zspp1qpya+8bafgectgltggn8aj6aicfoalcan+rt+jx9hv6oi6gtwqps/arkwxv6gmdkw=</latexit> <latexit sha1_base64="+cjv0oxg1vysmlcu4bihsyhivw8=">aaadkxicdzllbhmxfiad4vbclyulg6srulk0momqyinuwyzvvctsvoqtken4ehnfrrznsol4idjybjwbo+i2peiddfq1crzj0q//+z0en+nhzpmxcfynft25e+/+g72h9uepnzx91th/fmpuoqnteswvph9iqzmttguz5fq81xsliadnw+nhft8rqtzmys92nto+wgpjmkawdvba6lyhenlmy+is76yeiloi1lgv/pbdroanbr60/xkjylnk4bhmumqn1ojdvc7w+dfl5acdnppxk14x3bvjjzqgqpn0v/ytjrqpbjwwcgxml4lz23dyw0y49xvugjpjmsvj2gtsykfn360v7ugr4ixgpnry0sk1e3uhw8kyurigpmb2yrbzyvwx6xu2e9d3toafpzjch5qvhfofv12ei6ypsxwebcaahx+fzijdu2zodb2ojp1glbbyjhxisvru4i4s7zeb1bl3bocsyn04alqnpcmvozsxjemez96hm3hsfogmyv+b6wgr+oqmbhpx2cbllncsejjbboskllbbtajtxw9jt7ahvcto260kbiwf3zspp1qpya+8bafgectgltggn8aj6aicfoalcan+rt+jx9hv6oi6gtwqps/arkwxv6gmdkw=</latexit> <latexit sha1_base64="+cjv0oxg1vysmlcu4bihsyhivw8=">aaadkxicdzllbhmxfiad4vbclyulg6srulk0momqyinuwyzvvctsvoqtken4ehnfrrznsol4idjybjwbo+i2peiddfq1crzj0q//+z0en+nhzpmxcfynft25e+/+g72h9uepnzx91th/fmpuoqnteswvph9iqzmttguz5fq81xsliadnw+nhft8rqtzmys92nto+wgpjmkawdvba6lyhenlmy+is76yeiloi1lgv/pbdroanbr60/xkjylnk4bhmumqn1ojdvc7w+dfl5acdnppxk14x3bvjjzqgqpn0v/ytjrqpbjwwcgxml4lz23dyw0y49xvugjpjmsvj2gtsykfn360v7ugr4ixgpnry0sk1e3uhw8kyurigpmb2yrbzyvwx6xu2e9d3toafpzjch5qvhfofv12ei6ypsxwebcaahx+fzijdu2zodb2ojp1glbbyjhxisvru4i4s7zeb1bl3bocsyn04alqnpcmvozsxjemez96hm3hsfogmyv+b6wgr+oqmbhpx2cbllncsejjbboskllbbtajtxw9jt7ahvcto260kbiwf3zspp1qpya+8bafgectgltggn8aj6aicfoalcan+rt+jx9hv6oi6gtwqps/arkwxv6gmdkw=</latexit> NX 1 k kx VAE objective log p(~x i ~z ij, ) ) 1 KL (q (~z i ~x i ) p(~z i A i=1 j=1 N (g 1 (~x),h 2 (~x)) N (0,I) 1st term: 1 k = 1 k = 1 k NX i=1 X ij X ij kx j=1 log p(~x i ~z ij, ) ~z ij q (~z ~x i ) for 1 2 ~x 2 i f (~z ij ) 2 N (0,I) 1 2 ~x 2 i f H 2 (~x i ) 1 2 ~ ij + g 1 (~x i ) 2

8 <latexit sha1_base64="obuwvjnr4yfjbmwk5u7tlcxh820=">aaadcnicdzldbtmwfmedly8rvjq4gzegmqktw5vushcdnmhnrqyh0w1s3uao67smsr3ftmjn+hv4ml6a99gd4hrhwjm4uqs/zu9/tpjztpyltokg+o1tte7df/bw+5h/+mntz8/boy/olcxyqgdepjk/ilgikrn0ojlo6uwwu8zjlj7h8y8vpy9prpgu3/qyoyoop4iljgdtulh75yciku5ytexozdxcpaoegfbd3kr3kwjtjsd9u1qhchexeaitcokz1rjfbno9jgzkzizq2+6a748nuim2rlvjm3xd9r2c1ubwr7lqsb9ajfu+h7u7qs9yb7wrwlp0qb2n0y73c00kktgvmqryqweyzhpkck4zsan1uafohskct+nqsye5vsozhpyfey4zgynm3sm0xgdvvxjmlvry2dk51jpvzfxyx2xy6otjydcrfzokcv2ipeihlrdaapywnbkdlp3ajgfuwygzytcx7xbl+0jqh0ryjsxeiczka8bmmlr2e+g8s8amkzazm/ee1dkwkpokbtqkolm0srsbats6sej/ek5y7nj1pwsn5qsjy4wjjsvw0qsxnbhsancsnujoslqkw+0/bg77rjjr98kgf3593zn6xf/cnngf3oaucmehcasowskyaakuvf2v473dumrttl636rpz8uqal2ajwgd/aawtitk=</latexit> <latexit sha1_base64="obuwvjnr4yfjbmwk5u7tlcxh820=">aaadcnicdzldbtmwfmedly8rvjq4gzegmqktw5vushcdnmhnrqyh0w1s3uao67smsr3ftmjn+hv4ml6a99gd4hrhwjm4uqs/zu9/tpjztpyltokg+o1tte7df/bw+5h/+mntz8/boy/olcxyqgdepjk/ilgikrn0ojlo6uwwu8zjlj7h8y8vpy9prpgu3/qyoyoop4iljgdtulh75yciku5ytexozdxcpaoegfbd3kr3kwjtjsd9u1qhchexeaitcokz1rjfbno9jgzkzizq2+6a748nuim2rlvjm3xd9r2c1ubwr7lqsb9ajfu+h7u7qs9yb7wrwlp0qb2n0y73c00kktgvmqryqweyzhpkck4zsan1uafohskct+nqsye5vsozhpyfey4zgynm3sm0xgdvvxjmlvry2dk51jpvzfxyx2xy6otjydcrfzokcv2ipeihlrdaapywnbkdlp3ajgfuwygzytcx7xbl+0jqh0ryjsxeiczka8bmmlr2e+g8s8amkzazm/ee1dkwkpokbtqkolm0srsbats6sej/ek5y7nj1pwsn5qsjy4wjjsvw0qsxnbhsancsnujoslqkw+0/bg77rjjr98kgf3593zn6xf/cnngf3oaucmehcasowskyaakuvf2v473dumrttl636rpz8uqal2ajwgd/aawtitk=</latexit> <latexit sha1_base64="obuwvjnr4yfjbmwk5u7tlcxh820=">aaadcnicdzldbtmwfmedly8rvjq4gzegmqktw5vushcdnmhnrqyh0w1s3uao67smsr3ftmjn+hv4ml6a99gd4hrhwjm4uqs/zu9/tpjztpyltokg+o1tte7df/bw+5h/+mntz8/boy/olcxyqgdepjk/ilgikrn0ojlo6uwwu8zjlj7h8y8vpy9prpgu3/qyoyoop4iljgdtulh75yciku5ytexozdxcpaoegfbd3kr3kwjtjsd9u1qhchexeaitcokz1rjfbno9jgzkzizq2+6a748nuim2rlvjm3xd9r2c1ubwr7lqsb9ajfu+h7u7qs9yb7wrwlp0qb2n0y73c00kktgvmqryqweyzhpkck4zsan1uafohskct+nqsye5vsozhpyfey4zgynm3sm0xgdvvxjmlvry2dk51jpvzfxyx2xy6otjydcrfzokcv2ipeihlrdaapywnbkdlp3ajgfuwygzytcx7xbl+0jqh0ryjsxeiczka8bmmlr2e+g8s8amkzazm/ee1dkwkpokbtqkolm0srsbats6sej/ek5y7nj1pwsn5qsjy4wjjsvw0qsxnbhsancsnujoslqkw+0/bg77rjjr98kgf3593zn6xf/cnngf3oaucmehcasowskyaakuvf2v473dumrttl636rpz8uqal2ajwgd/aawtitk=</latexit> <latexit sha1_base64="obuwvjnr4yfjbmwk5u7tlcxh820=">aaadcnicdzldbtmwfmedly8rvjq4gzegmqktw5vushcdnmhnrqyh0w1s3uao67smsr3ftmjn+hv4ml6a99gd4hrhwjm4uqs/zu9/tpjztpyltokg+o1tte7df/bw+5h/+mntz8/boy/olcxyqgdepjk/ilgikrn0ojlo6uwwu8zjlj7h8y8vpy9prpgu3/qyoyoop4iljgdtulh75yciku5ytexozdxcpaoegfbd3kr3kwjtjsd9u1qhchexeaitcokz1rjfbno9jgzkzizq2+6a748nuim2rlvjm3xd9r2c1ubwr7lqsb9ajfu+h7u7qs9yb7wrwlp0qb2n0y73c00kktgvmqryqweyzhpkck4zsan1uafohskct+nqsye5vsozhpyfey4zgynm3sm0xgdvvxjmlvry2dk51jpvzfxyx2xy6otjydcrfzokcv2ipeihlrdaapywnbkdlp3ajgfuwygzytcx7xbl+0jqh0ryjsxeiczka8bmmlr2e+g8s8amkzazm/ee1dkwkpokbtqkolm0srsbats6sej/ek5y7nj1pwsn5qsjy4wjjsvw0qsxnbhsancsnujoslqkw+0/bg77rjjr98kgf3593zn6xf/cnngf3oaucmehcasowskyaakuvf2v473dumrttl636rpz8uqal2ajwgd/aawtitk=</latexit> <latexit sha1_base64="18pwsf9dxoehzbhnnwwvjwo+mci=">aaac9xicdzllbhmxfiad4vagwwllbiwijdzemxuslcvyskqkrnpkmttyogcau99ke5kmlh+lo1swvagvwnvusqdejnaks7/p9x/fjkvnmxvz9qut3ll77/6dnyfpo8dpnj7r9p4fwvubckoqudinjbhamyshy47ditzarmnhudz/uolhczcwkfnfltwmbtmtrgkuujiadhufl0bbxupes6+hcjnupxtk68dbim9ehzvxool1fhztrwsb0lforb3lmxzjt4xjlenii9qcjvscnmeoskke2lffnz3gvzezxzuycuih19m/kzwr1i5fgz2cujlts1xyx2xuu+r92dopawes3m5u1rw7hvcpgafmahv8gqwhhswzyjojhlaxnytncwklqoqgcuoljufbn/o3eqibwbkdvd8tnnihbogdidjqzwhtpitmebv8ucx1fr783tzcfyzuv54diyjf3zqs7g/aeh4r6rwovmidywzctkhsfin+f4kqxv7n7w5vi6o9qz4n8s9v+/sfmp+wg3brs/qa5egd2kef0ceaioow6apdo+/jirlkvixxt9ak09s8qbur/lgbavn4ia==</latexit> <latexit sha1_base64="18pwsf9dxoehzbhnnwwvjwo+mci=">aaac9xicdzllbhmxfiad4vagwwllbiwijdzemxuslcvyskqkrnpkmttyogcau99ke5kmlh+lo1swvagvwnvusqdejnaks7/p9x/fjkvnmxvz9qut3ll77/6dnyfpo8dpnj7r9p4fwvubckoqudinjbhamyshy47ditzarmnhudz/uolhczcwkfnfltwmbtmtrgkuujiadhufl0bbxupes6+hcjnupxtk68dbim9ehzvxool1fhztrwsb0lforb3lmxzjt4xjlenii9qcjvscnmeoskke2lffnz3gvzezxzuycuih19m/kzwr1i5fgz2cujlts1xyx2xuu+r92dopawes3m5u1rw7hvcpgafmahv8gqwhhswzyjojhlaxnytncwklqoqgcuoljufbn/o3eqibwbkdvd8tnnihbogdidjqzwhtpitmebv8ucx1fr783tzcfyzuv54diyjf3zqs7g/aeh4r6rwovmidywzctkhsfin+f4kqxv7n7w5vi6o9qz4n8s9v+/sfmp+wg3brs/qa5egd2kef0ceaioow6apdo+/jirlkvixxt9ak09s8qbur/lgbavn4ia==</latexit> <latexit sha1_base64="18pwsf9dxoehzbhnnwwvjwo+mci=">aaac9xicdzllbhmxfiad4vagwwllbiwijdzemxuslcvyskqkrnpkmttyogcau99ke5kmlh+lo1swvagvwnvusqdejnaks7/p9x/fjkvnmxvz9qut3ll77/6dnyfpo8dpnj7r9p4fwvubckoqudinjbhamyshy47ditzarmnhudz/uolhczcwkfnfltwmbtmtrgkuujiadhufl0bbxupes6+hcjnupxtk68dbim9ehzvxool1fhztrwsb0lforb3lmxzjt4xjlenii9qcjvscnmeoskke2lffnz3gvzezxzuycuih19m/kzwr1i5fgz2cujlts1xyx2xuu+r92dopawes3m5u1rw7hvcpgafmahv8gqwhhswzyjojhlaxnytncwklqoqgcuoljufbn/o3eqibwbkdvd8tnnihbogdidjqzwhtpitmebv8ucx1fr783tzcfyzuv54diyjf3zqs7g/aeh4r6rwovmidywzctkhsfin+f4kqxv7n7w5vi6o9qz4n8s9v+/sfmp+wg3brs/qa5egd2kef0ceaioow6apdo+/jirlkvixxt9ak09s8qbur/lgbavn4ia==</latexit> <latexit sha1_base64="18pwsf9dxoehzbhnnwwvjwo+mci=">aaac9xicdzllbhmxfiad4vagwwllbiwijdzemxuslcvyskqkrnpkmttyogcau99ke5kmlh+lo1swvagvwnvusqdejnaks7/p9x/fjkvnmxvz9qut3ll77/6dnyfpo8dpnj7r9p4fwvubckoqudinjbhamyshy47ditzarmnhudz/uolhczcwkfnfltwmbtmtrgkuujiadhufl0bbxupes6+hcjnupxtk68dbim9ehzvxool1fhztrwsb0lforb3lmxzjt4xjlenii9qcjvscnmeoskke2lffnz3gvzezxzuycuih19m/kzwr1i5fgz2cujlts1xyx2xuu+r92dopawes3m5u1rw7hvcpgafmahv8gqwhhswzyjojhlaxnytncwklqoqgcuoljufbn/o3eqibwbkdvd8tnnihbogdidjqzwhtpitmebv8ucx1fr783tzcfyzuv54diyjf3zqs7g/aeh4r6rwovmidywzctkhsfin+f4kqxv7n7w5vi6o9qz4n8s9v+/sfmp+wg3brs/qa5egd2kef0ceaioow6apdo+/jirlkvixxt9ak09s8qbur/lgbavn4ia==</latexit> <latexit sha1_base64="afxxcjjdeqmrsoztm3pnejpg5ho=">aaac+nicdzlnbhmxemed5asnh01b4slfiklianfuhathci6cqikrtli2jbzobneq17bs2ddu3zfpdzujj8el8dz10huigxjj8l/z+9vjsz0zkrzg8e9odofuvfsptra7dx89frlt23166hrloyy4ltoez8ybfapgkfdcsbhaykzcuxb2acmp5mcd0oorlgxmsnaqrc44w5ca9p6nwacynzq1hzgmztyc9vrxif4f3rrji/qkiyppbudxoto8kkehl8y5cribnhhmuxajdtethbjgz9gpjinuraq38asgavoqzgy01zymhxsv/xufz6vzizilzpjh4dpsmfwxg1eyf5h4ouyfophtobysfdvd3gadcqsc5siixq0iz6w8yjzxdhfw7aykvnfdlkznfcruvpyn/m1s1+saram9p0lrmzquogizcky7wldpbyxma5/myx+f1h64sdefc/7xs2/lwjedcib9fhvpzwoda6fnbxdrgis2aomalayppfz1n7x/0n7ttxe4hctxipnyrr/3sfkjw+qfeulek4s8j3vkmzkgi8ljjbki1+rhdbldrd+j61tr1gnwpcnref28avyh+ic=</latexit> <latexit sha1_base64="afxxcjjdeqmrsoztm3pnejpg5ho=">aaac+nicdzlnbhmxemed5asnh01b4slfiklianfuhathci6cqikrtli2jbzobneq17bs2ddu3zfpdzujj8el8dz10huigxjj8l/z+9vjsz0zkrzg8e9odofuvfsptra7dx89frlt23166hrloyy4ltoez8ybfapgkfdcsbhaykzcuxb2acmp5mcd0oorlgxmsnaqrc44w5ca9p6nwacynzq1hzgmztyc9vrxif4f3rrji/qkiyppbudxoto8kkehl8y5cribnhhmuxajdtethbjgz9gpjinuraq38asgavoqzgy01zymhxsv/xufz6vzizilzpjh4dpsmfwxg1eyf5h4ouyfophtobysfdvd3gadcqsc5siixq0iz6w8yjzxdhfw7aykvnfdlkznfcruvpyn/m1s1+saram9p0lrmzquogizcky7wldpbyxma5/myx+f1h64sdefc/7xs2/lwjedcib9fhvpzwoda6fnbxdrgis2aomalayppfz1n7x/0n7ttxe4hctxipnyrr/3sfkjw+qfeulek4s8j3vkmzkgi8ljjbki1+rhdbldrd+j61tr1gnwpcnref28avyh+ic=</latexit> <latexit sha1_base64="afxxcjjdeqmrsoztm3pnejpg5ho=">aaac+nicdzlnbhmxemed5asnh01b4slfiklianfuhathci6cqikrtli2jbzobneq17bs2ddu3zfpdzujj8el8dz10huigxjj8l/z+9vjsz0zkrzg8e9odofuvfsptra7dx89frlt23166hrloyy4ltoez8ybfapgkfdcsbhaykzcuxb2acmp5mcd0oorlgxmsnaqrc44w5ca9p6nwacynzq1hzgmztyc9vrxif4f3rrji/qkiyppbudxoto8kkehl8y5cribnhhmuxajdtethbjgz9gpjinuraq38asgavoqzgy01zymhxsv/xufz6vzizilzpjh4dpsmfwxg1eyf5h4ouyfophtobysfdvd3gadcqsc5siixq0iz6w8yjzxdhfw7aykvnfdlkznfcruvpyn/m1s1+saram9p0lrmzquogizcky7wldpbyxma5/myx+f1h64sdefc/7xs2/lwjedcib9fhvpzwoda6fnbxdrgis2aomalayppfz1n7x/0n7ttxe4hctxipnyrr/3sfkjw+qfeulek4s8j3vkmzkgi8ljjbki1+rhdbldrd+j61tr1gnwpcnref28avyh+ic=</latexit> <latexit sha1_base64="afxxcjjdeqmrsoztm3pnejpg5ho=">aaac+nicdzlnbhmxemed5asnh01b4slfiklianfuhathci6cqikrtli2jbzobneq17bs2ddu3zfpdzujj8el8dz10huigxjj8l/z+9vjsz0zkrzg8e9odofuvfsptra7dx89frlt23166hrloyy4ltoez8ybfapgkfdcsbhaykzcuxb2acmp5mcd0oorlgxmsnaqrc44w5ca9p6nwacynzq1hzgmztyc9vrxif4f3rrji/qkiyppbudxoto8kkehl8y5cribnhhmuxajdtethbjgz9gpjinuraq38asgavoqzgy01zymhxsv/xufz6vzizilzpjh4dpsmfwxg1eyf5h4ouyfophtobysfdvd3gadcqsc5siixq0iz6w8yjzxdhfw7aykvnfdlkznfcruvpyn/m1s1+saram9p0lrmzquogizcky7wldpbyxma5/myx+f1h64sdefc/7xs2/lwjedcib9fhvpzwoda6fnbxdrgis2aomalayppfz1n7x/0n7ttxe4hctxipnyrr/3sfkjw+qfeulek4s8j3vkmzkgi8ljjbki1+rhdbldrd+j61tr1gnwpcnref28avyh+ic=</latexit> <latexit sha1_base64="hsgx9mc5p4dsv4w8avwbzlla5ik=">aaac7hicdzlnbhmxemed5assxy0cuvhesewcaldcgmmff7hursjtrtatvm5sy9vrw/zsagr5lxpd5cjt8ak8du66qmqdi1n6a35/2zmel0ykh1n2q5fcuhnr9p2nu+m9+w8eptrcenzgdgm5dlmw2h6vzieucoyoumkrscdqusjhefz+wq9nyj3q6jpodyxrdqpejtjdmpps7nl6o3tjp7442exng2wzdf3kreitnvzptno/i4nmtq0kuwtojflm4ngzi4jlcgnrodcmn7ftgewpwa1u7jclb/o8zia00jyuhxsz/xuhz7vz87qmzprh1hxzivkvnmqwejv2qpkgqfhri6pgutr00t+dcasc5twkxq2itvi+zzzxjk+upowcr1zxnvmtxwg1c/7yv8pdwavotfd+uebtqrycomxkge1g1ayvtguvffhf830e/m7aqx8s+f9phfhki045k36vi2fnkc6a0/muugjbrreacs0qohrsq5dg+efdaa+lg51bng3yt6/7u+/an7bbnpjnzjvk5a3zjr/iphksths5jffke6ksy+rbcnvttxrtnidkjzifvwhp5pmy</latexit> <latexit sha1_base64="hsgx9mc5p4dsv4w8avwbzlla5ik=">aaac7hicdzlnbhmxemed5assxy0cuvhesewcaldcgmmff7hursjtrtatvm5sy9vrw/zsagr5lxpd5cjt8ak8du66qmqdi1n6a35/2zmel0ykh1n2q5fcuhnr9p2nu+m9+w8eptrcenzgdgm5dlmw2h6vzieucoyoumkrscdqusjhefz+wq9nyj3q6jpodyxrdqpejtjdmpps7nl6o3tjp7442exng2wzdf3kreitnvzptno/i4nmtq0kuwtojflm4ngzi4jlcgnrodcmn7ftgewpwa1u7jclb/o8zia00jyuhxsz/xuhz7vz87qmzprh1hxzivkvnmqwejv2qpkgqfhri6pgutr00t+dcasc5twkxq2itvi+zzzxjk+upowcr1zxnvmtxwg1c/7yv8pdwavotfd+uebtqrycomxkge1g1ayvtguvffhf830e/m7aqx8s+f9phfhki045k36vi2fnkc6a0/muugjbrreacs0qohrsq5dg+efdaa+lg51bng3yt6/7u+/an7bbnpjnzjvk5a3zjr/iphksths5jffke6ksy+rbcnvttxrtnidkjzifvwhp5pmy</latexit> <latexit sha1_base64="hsgx9mc5p4dsv4w8avwbzlla5ik=">aaac7hicdzlnbhmxemed5assxy0cuvhesewcaldcgmmff7hursjtrtatvm5sy9vrw/zsagr5lxpd5cjt8ak8du66qmqdi1n6a35/2zmel0ykh1n2q5fcuhnr9p2nu+m9+w8eptrcenzgdgm5dlmw2h6vzieucoyoumkrscdqusjhefz+wq9nyj3q6jpodyxrdqpejtjdmpps7nl6o3tjp7442exng2wzdf3kreitnvzptno/i4nmtq0kuwtojflm4ngzi4jlcgnrodcmn7ftgewpwa1u7jclb/o8zia00jyuhxsz/xuhz7vz87qmzprh1hxzivkvnmqwejv2qpkgqfhri6pgutr00t+dcasc5twkxq2itvi+zzzxjk+upowcr1zxnvmtxwg1c/7yv8pdwavotfd+uebtqrycomxkge1g1ayvtguvffhf830e/m7aqx8s+f9phfhki045k36vi2fnkc6a0/muugjbrreacs0qohrsq5dg+efdaa+lg51bng3yt6/7u+/an7bbnpjnzjvk5a3zjr/iphksths5jffke6ksy+rbcnvttxrtnidkjzifvwhp5pmy</latexit> <latexit sha1_base64="hsgx9mc5p4dsv4w8avwbzlla5ik=">aaac7hicdzlnbhmxemed5assxy0cuvhesewcaldcgmmff7hursjtrtatvm5sy9vrw/zsagr5lxpd5cjt8ak8du66qmqdi1n6a35/2zmel0ykh1n2q5fcuhnr9p2nu+m9+w8eptrcenzgdgm5dlmw2h6vzieucoyoumkrscdqusjhefz+wq9nyj3q6jpodyxrdqpejtjdmpps7nl6o3tjp7442exng2wzdf3kreitnvzptno/i4nmtq0kuwtojflm4ngzi4jlcgnrodcmn7ftgewpwa1u7jclb/o8zia00jyuhxsz/xuhz7vz87qmzprh1hxzivkvnmqwejv2qpkgqfhri6pgutr00t+dcasc5twkxq2itvi+zzzxjk+upowcr1zxnvmtxwg1c/7yv8pdwavotfd+uebtqrycomxkge1g1ayvtguvffhf830e/m7aqx8s+f9phfhki045k36vi2fnkc6a0/muugjbrreacs0qohrsq5dg+efdaa+lg51bng3yt6/7u+/an7bbnpjnzjvk5a3zjr/iphksths5jffke6ksy+rbcnvttxrtnidkjzifvwhp5pmy</latexit> VAE objective Learning Algorithm: 1. sample { ij } for i = 1:N, j = 1:k 2. compute gradient w.r.t., 1, 2 3. take small step in direction of gradient 4. repeat to convergence = 1 k X ij This is the key expression! N (0,I) 1 2 ~x 2 i f H 2 (~x i ) 1 2 ~ ij + g 1 (~x i ) 2

9 <latexit sha1_base64="ekmzyjathqf2varkxw+nbpcajqg=">aaadknicdzllbhmxfiad4vags1nysrgikfijopkkczyvsihvvstsvsqkken4mly9trhphctgb9rhydvdvzy8ce4yqkgcr7l163y/b+d4qaw3kctxjejo3xv3h+w8jb89fvj0t7n37msqyldwpuooczyklgkuwrc4chamdspluldt4cxhbt+dmgo5kl9hplm/jgpjc04jhnsg+snjevc39vxhpgu+jzohxmfiofrwlhtcnw5pxwckbqoyj7279xqcgt4uoi9h80gzlxsszebtkdaiheo4huw1fmyjraussaccwntlew19rwxwkpips8oytegfgbnekjkuzpbd8seevwqzec6vcsncf5n9e4ujpbwzchicjyhcbrjf8l+sv0h+vu+41buwsvch5zxaopcienjedamgzkeqani4k6yfcwwcuom4zit7tlvzejlyobat787dm9t7dqbge+fom1dahymwgcktvpat25rkhci9w7uh9e5ga6nbc/zxc2tkwbcvpus4o008mqy6yrrpn8g8bvnnwlsturykf0r6opq/3ez2tjg56krjj/3ytnx4of4jo+gfeonakexv0ch6ji5rf1f0ia7qdfovxuzx0xv0s7jgjxrnc7qw0e8/imumra==</latexit> <latexit sha1_base64="ekmzyjathqf2varkxw+nbpcajqg=">aaadknicdzllbhmxfiad4vags1nysrgikfijopkkczyvsihvvstsvsqkken4mly9trhphctgb9rhydvdvzy8ce4yqkgcr7l163y/b+d4qaw3kctxjejo3xv3h+w8jb89fvj0t7n37msqyldwpuooczyklgkuwrc4chamdspluldt4cxhbt+dmgo5kl9hplm/jgpjc04jhnsg+snjevc39vxhpgu+jzohxmfiofrwlhtcnw5pxwckbqoyj7279xqcgt4uoi9h80gzlxsszebtkdaiheo4huw1fmyjraussaccwntlew19rwxwkpips8oytegfgbnekjkuzpbd8seevwqzec6vcsncf5n9e4ujpbwzchicjyhcbrjf8l+sv0h+vu+41buwsvch5zxaopcienjedamgzkeqani4k6yfcwwcuom4zit7tlvzejlyobat787dm9t7dqbge+fom1dahymwgcktvpat25rkhci9w7uh9e5ga6nbc/zxc2tkwbcvpus4o008mqy6yrrpn8g8bvnnwlsturykf0r6opq/3ez2tjg56krjj/3ytnx4of4jo+gfeonakexv0ch6ji5rf1f0ia7qdfovxuzx0xv0s7jgjxrnc7qw0e8/imumra==</latexit> <latexit sha1_base64="ekmzyjathqf2varkxw+nbpcajqg=">aaadknicdzllbhmxfiad4vags1nysrgikfijopkkczyvsihvvstsvsqkken4mly9trhphctgb9rhydvdvzy8ce4yqkgcr7l163y/b+d4qaw3kctxjejo3xv3h+w8jb89fvj0t7n37msqyldwpuooczyklgkuwrc4chamdspluldt4cxhbt+dmgo5kl9hplm/jgpjc04jhnsg+snjevc39vxhpgu+jzohxmfiofrwlhtcnw5pxwckbqoyj7279xqcgt4uoi9h80gzlxsszebtkdaiheo4huw1fmyjraussaccwntlew19rwxwkpips8oytegfgbnekjkuzpbd8seevwqzec6vcsncf5n9e4ujpbwzchicjyhcbrjf8l+sv0h+vu+41buwsvch5zxaopcienjedamgzkeqani4k6yfcwwcuom4zit7tlvzejlyobat787dm9t7dqbge+fom1dahymwgcktvpat25rkhci9w7uh9e5ga6nbc/zxc2tkwbcvpus4o008mqy6yrrpn8g8bvnnwlsturykf0r6opq/3ez2tjg56krjj/3ytnx4of4jo+gfeonakexv0ch6ji5rf1f0ia7qdfovxuzx0xv0s7jgjxrnc7qw0e8/imumra==</latexit> <latexit sha1_base64="ekmzyjathqf2varkxw+nbpcajqg=">aaadknicdzllbhmxfiad4vags1nysrgikfijopkkczyvsihvvstsvsqkken4mly9trhphctgb9rhydvdvzy8ce4yqkgcr7l163y/b+d4qaw3kctxjejo3xv3h+w8jb89fvj0t7n37msqyldwpuooczyklgkuwrc4chamdspluldt4cxhbt+dmgo5kl9hplm/jgpjc04jhnsg+snjevc39vxhpgu+jzohxmfiofrwlhtcnw5pxwckbqoyj7279xqcgt4uoi9h80gzlxsszebtkdaiheo4huw1fmyjraussaccwntlew19rwxwkpips8oytegfgbnekjkuzpbd8seevwqzec6vcsncf5n9e4ujpbwzchicjyhcbrjf8l+sv0h+vu+41buwsvch5zxaopcienjedamgzkeqani4k6yfcwwcuom4zit7tlvzejlyobat787dm9t7dqbge+fom1dahymwgcktvpat25rkhci9w7uh9e5ga6nbc/zxc2tkwbcvpus4o008mqy6yrrpn8g8bvnnwlsturykf0r6opq/3ez2tjg56krjj/3ytnx4of4jo+gfeonakexv0ch6ji5rf1f0ia7qdfovxuzx0xv0s7jgjxrnc7qw0e8/imumra==</latexit> <latexit sha1_base64="kiutymychhvxnfqehhyd8uzwuja=">aaac9hicdzlnbhmxemed5anl+urhymuiqiohot0kcy4vxdhvrsjtpwwaoc5sytvrw/bsnqnzn+knlsnpwgvwnjjpcpenjgtpr/n9pbznpdfsoeysx53ozt1793d2h8qphz1+8rs79+ze6djyghattt2bmadskbigqalnxgirjhjojxcfv/y0auuevl9wawbusjksueamq2rc7wzsz6jzx3znca7ixo+7vasfrinui7qrpdle8xiv8zobal4wojbl5twwtqyoplmouiq6zkohhvelnonhkiov4ez+ffwavgqzkc21dushxwf/3ufz4dyymarnwxdu2myv/bcblpi/h3mhtimg+o1beskparrqa50kcxzlmgjgrqh3pxzolomyuhxhmyjlroucqanphkpqf+7fphw9cdca2vvzdlwpwxeulvpoaaebnq1glvpaz3mo79pah2wv+mpb/3qobbh46qwcsx/uxtui0ao4xbtbvqou2gcma1awpjba1xgyf9qe9ry4oeinst/9/lz3+kh5cbvkbxlj9klk3pfd8okckwhhpclx5iz8j6roovow3dxao06z5znziojhb5tx9ss=</latexit> <latexit sha1_base64="kiutymychhvxnfqehhyd8uzwuja=">aaac9hicdzlnbhmxemed5anl+urhymuiqiohot0kcy4vxdhvrsjtpwwaoc5sytvrw/bsnqnzn+knlsnpwgvwnjjpcpenjgtpr/n9pbznpdfsoeysx53ozt1793d2h8qphz1+8rs79+ze6djyghattt2bmadskbigqalnxgirjhjojxcfv/y0auuevl9wawbusjksueamq2rc7wzsz6jzx3znca7ixo+7vasfrinui7qrpdle8xiv8zobal4wojbl5twwtqyoplmouiq6zkohhvelnonhkiov4ez+ffwavgqzkc21dushxwf/3ufz4dyymarnwxdu2myv/bcblpi/h3mhtimg+o1beskparrqa50kcxzlmgjgrqh3pxzolomyuhxhmyjlroucqanphkpqf+7fphw9cdca2vvzdlwpwxeulvpoaaebnq1glvpaz3mo79pah2wv+mpb/3qobbh46qwcsx/uxtui0ao4xbtbvqou2gcma1awpjba1xgyf9qe9ry4oeinst/9/lz3+kh5cbvkbxlj9klk3pfd8okckwhhpclx5iz8j6roovow3dxao06z5znziojhb5tx9ss=</latexit> <latexit sha1_base64="kiutymychhvxnfqehhyd8uzwuja=">aaac9hicdzlnbhmxemed5anl+urhymuiqiohot0kcy4vxdhvrsjtpwwaoc5sytvrw/bsnqnzn+knlsnpwgvwnjjpcpenjgtpr/n9pbznpdfsoeysx53ozt1793d2h8qphz1+8rs79+ze6djyghattt2bmadskbigqalnxgirjhjojxcfv/y0auuevl9wawbusjksueamq2rc7wzsz6jzx3znca7ixo+7vasfrinui7qrpdle8xiv8zobal4wojbl5twwtqyoplmouiq6zkohhvelnonhkiov4ez+ffwavgqzkc21dushxwf/3ufz4dyymarnwxdu2myv/bcblpi/h3mhtimg+o1beskparrqa50kcxzlmgjgrqh3pxzolomyuhxhmyjlroucqanphkpqf+7fphw9cdca2vvzdlwpwxeulvpoaaebnq1glvpaz3mo79pah2wv+mpb/3qobbh46qwcsx/uxtui0ao4xbtbvqou2gcma1awpjba1xgyf9qe9ry4oeinst/9/lz3+kh5cbvkbxlj9klk3pfd8okckwhhpclx5iz8j6roovow3dxao06z5znziojhb5tx9ss=</latexit> <latexit sha1_base64="kiutymychhvxnfqehhyd8uzwuja=">aaac9hicdzlnbhmxemed5anl+urhymuiqiohot0kcy4vxdhvrsjtpwwaoc5sytvrw/bsnqnzn+knlsnpwgvwnjjpcpenjgtpr/n9pbznpdfsoeysx53ozt1793d2h8qphz1+8rs79+ze6djyghattt2bmadskbigqalnxgirjhjojxcfv/y0auuevl9wawbusjksueamq2rc7wzsz6jzx3znca7ixo+7vasfrinui7qrpdle8xiv8zobal4wojbl5twwtqyoplmouiq6zkohhvelnonhkiov4ez+ffwavgqzkc21dushxwf/3ufz4dyymarnwxdu2myv/bcblpi/h3mhtimg+o1beskparrqa50kcxzlmgjgrqh3pxzolomyuhxhmyjlroucqanphkpqf+7fphw9cdca2vvzdlwpwxeulvpoaaebnq1glvpaz3mo79pah2wv+mpb/3qobbh46qwcsx/uxtui0ao4xbtbvqou2gcma1awpjba1xgyf9qe9ry4oeinst/9/lz3+kh5cbvkbxlj9klk3pfd8okckwhhpclx5iz8j6roovow3dxao06z5znziojhb5tx9ss=</latexit> Terminology Z ELBO = evidence lower bound q(z ) log apple p(x, z ) q(z ) dz (Same thing we ve been calling neg. free energy ) which is a lower bound on log p(x ) Variational EM = general family of algorithms in which q(z ) 6= p(z x, ) so in E step we minimize but can t set it to zero KL(q(z ) p(z x, )) often refer to q(z) as variational approximation to p(z x)

10 <latexit sha1_base64="ixsrnj0wa+0hzlx4ci9qdaljs44=">aaac7xicdzlnahsxemfl7ve6/urayy9ltse91oygqhsm6awnkekdblyokexzwerfslnobkhhyk2kx75mx6bvu9lzsr1ubwr/5vfxsjrr2ajumm9/dzj79x88flt1oh3y9nnzf9s7l0+cri2dptnc27mxdsc4gj5yfhbmlfa5fna6vvy05kczsi5r9rxnboasxihecuyxpgbvqmqpin1dvbttd/nevopsuxsn6jimjkc7nz/lrlnagkimqhodijc49nqizwjcwtyodgwx9aigusoqwq396s4hexszk6zsni6f2sr79w5ppxnzoy5osxhq2myz/bcb1fh9hhquti2g2n1bvs0y1nmyadmew2ao5lfqznm8a8am1fkgsu1pwiq4ylpkqia+5gow/ll/x4swdtca4p15idqeeesglipntin1m1ywfvxwzrxr+yl4vy1cfyz4x8+rlzevx8qo8edtpluodaysu26drqmwbqdgnaimkgutqhrnx7snvslo9npf3iu+7hcpdpufsevekzdklxtkazkgn8kx6rngnlkht+r7opob5ftye2dnos2ev2qtkh+/axpg9iu=</latexit> <latexit sha1_base64="ixsrnj0wa+0hzlx4ci9qdaljs44=">aaac7xicdzlnahsxemfl7ve6/urayy9ltse91oygqhsm6awnkekdblyokexzwerfslnobkhhyk2kx75mx6bvu9lzsr1ubwr/5vfxsjrr2ajumm9/dzj79x88flt1oh3y9nnzf9s7l0+cri2dptnc27mxdsc4gj5yfhbmlfa5fna6vvy05kczsi5r9rxnboasxihecuyxpgbvqmqpin1dvbttd/nevopsuxsn6jimjkc7nz/lrlnagkimqhodijc49nqizwjcwtyodgwx9aigusoqwq396s4hexszk6zsni6f2sr79w5ppxnzoy5osxhq2myz/bcb1fh9hhquti2g2n1bvs0y1nmyadmew2ao5lfqznm8a8am1fkgsu1pwiq4ylpkqia+5gow/ll/x4swdtca4p15idqeeesglipntin1m1ywfvxwzrxr+yl4vy1cfyz4x8+rlzevx8qo8edtpluodaysu26drqmwbqdgnaimkgutqhrnx7snvslo9npf3iu+7hcpdpufsevekzdklxtkazkgn8kx6rngnlkht+r7opob5ftye2dnos2ev2qtkh+/axpg9iu=</latexit> <latexit sha1_base64="ixsrnj0wa+0hzlx4ci9qdaljs44=">aaac7xicdzlnahsxemfl7ve6/urayy9ltse91oygqhsm6awnkekdblyokexzwerfslnobkhhyk2kx75mx6bvu9lzsr1ubwr/5vfxsjrr2ajumm9/dzj79x88flt1oh3y9nnzf9s7l0+cri2dptnc27mxdsc4gj5yfhbmlfa5fna6vvy05kczsi5r9rxnboasxihecuyxpgbvqmqpin1dvbttd/nevopsuxsn6jimjkc7nz/lrlnagkimqhodijc49nqizwjcwtyodgwx9aigusoqwq396s4hexszk6zsni6f2sr79w5ppxnzoy5osxhq2myz/bcb1fh9hhquti2g2n1bvs0y1nmyadmew2ao5lfqznm8a8am1fkgsu1pwiq4ylpkqia+5gow/ll/x4swdtca4p15idqeeesglipntin1m1ywfvxwzrxr+yl4vy1cfyz4x8+rlzevx8qo8edtpluodaysu26drqmwbqdgnaimkgutqhrnx7snvslo9npf3iu+7hcpdpufsevekzdklxtkazkgn8kx6rngnlkht+r7opob5ftye2dnos2ev2qtkh+/axpg9iu=</latexit> <latexit sha1_base64="ixsrnj0wa+0hzlx4ci9qdaljs44=">aaac7xicdzlnahsxemfl7ve6/urayy9ltse91oygqhsm6awnkekdblyokexzwerfslnobkhhyk2kx75mx6bvu9lzsr1ubwr/5vfxsjrr2ajumm9/dzj79x88flt1oh3y9nnzf9s7l0+cri2dptnc27mxdsc4gj5yfhbmlfa5fna6vvy05kczsi5r9rxnboasxihecuyxpgbvqmqpin1dvbttd/nevopsuxsn6jimjkc7nz/lrlnagkimqhodijc49nqizwjcwtyodgwx9aigusoqwq396s4hexszk6zsni6f2sr79w5ppxnzoy5osxhq2myz/bcb1fh9hhquti2g2n1bvs0y1nmyadmew2ao5lfqznm8a8am1fkgsu1pwiq4ylpkqia+5gow/ll/x4swdtca4p15idqeeesglipntin1m1ywfvxwzrxr+yl4vy1cfyz4x8+rlzevx8qo8edtpluodaysu26drqmwbqdgnaimkgutqhrnx7snvslo9npf3iu+7hcpdpufsevekzdklxtkazkgn8kx6rngnlkht+r7opob5ftye2dnos2ev2qtkh+/axpg9iu=</latexit> Results [Kingma & Welling 2014] f (z) 2D latent z

11 Results [Kingma & Welling 2014]

12 <latexit sha1_base64="udqcuvokqf5ylbd81gwglcbf5b8=">aaac83icdzllbhmxfiad4vags1nyshkrizuf0uyfbmuqbfhvrsjtpuw68jhnequ+yfzmk1p+ku5qwfimvabvg5ooejnaksz9ot/vy/scl4pry9p0vy+6d//bw0c7j+mnt5893+3vvtg1styerkqyqc9lbibrasnllynzpqhzksfzeflpxc8a0izk8duufuw4nglauyjtsbx93vnhcjwnreb382bxtugp0mg6jmrbzk0yodzoir3ez3wqsc1bwmkwmemsvxbislaumpbxxhtqmfzigyydfjidmbj1zx3yjmsmssv1wmim6+zfoxzmxix5gzwc27npslxyx2xc2+rjxfghaguc3b1u1syxmlm1izlsdcsyzrcyabrumpa51pjy0kw4zgvceck5floxu9f4d+hezd5vaquvd+4it1l5cerinrzgsqobnilgzirv8irud5l3b1uf/ljsfz3hmge+einbzb13cbmitagsllrgugxxxqdktcgpkjipfbzmn3wnvs1od4zzosy+vb8chru/yqe9qq/rpsrqb3sipqmtneie1egg3alvur3drn+i2ztr1gv3veqbef34dx2/9mi=</latexit> <latexit sha1_base64="udqcuvokqf5ylbd81gwglcbf5b8=">aaac83icdzllbhmxfiad4vags1nyshkrizuf0uyfbmuqbfhvrsjtpuw68jhnequ+yfzmk1p+ku5qwfimvabvg5ooejnaksz9ot/vy/scl4pry9p0vy+6d//bw0c7j+mnt5893+3vvtg1styerkqyqc9lbibrasnllynzpqhzksfzeflpxc8a0izk8duufuw4nglauyjtsbx93vnhcjwnreb382bxtugp0mg6jmrbzk0yodzoir3ez3wqsc1bwmkwmemsvxbislaumpbxxhtqmfzigyydfjidmbj1zx3yjmsmssv1wmim6+zfoxzmxix5gzwc27npslxyx2xc2+rjxfghaguc3b1u1syxmlm1izlsdcsyzrcyabrumpa51pjy0kw4zgvceck5floxu9f4d+hezd5vaquvd+4it1l5cerinrzgsqobnilgzirv8irud5l3b1uf/ljsfz3hmge+einbzb13cbmitagsllrgugxxxqdktcgpkjipfbzmn3wnvs1od4zzosy+vb8chru/yqe9qq/rpsrqb3sipqmtneie1egg3alvur3drn+i2ztr1gv3veqbef34dx2/9mi=</latexit> <latexit sha1_base64="udqcuvokqf5ylbd81gwglcbf5b8=">aaac83icdzllbhmxfiad4vags1nyshkrizuf0uyfbmuqbfhvrsjtpuw68jhnequ+yfzmk1p+ku5qwfimvabvg5ooejnaksz9ot/vy/scl4pry9p0vy+6d//bw0c7j+mnt5893+3vvtg1styerkqyqc9lbibrasnllynzpqhzksfzeflpxc8a0izk8duufuw4nglauyjtsbx93vnhcjwnreb382bxtugp0mg6jmrbzk0yodzoir3ez3wqsc1bwmkwmemsvxbislaumpbxxhtqmfzigyydfjidmbj1zx3yjmsmssv1wmim6+zfoxzmxix5gzwc27npslxyx2xc2+rjxfghaguc3b1u1syxmlm1izlsdcsyzrcyabrumpa51pjy0kw4zgvceck5floxu9f4d+hezd5vaquvd+4it1l5cerinrzgsqobnilgzirv8irud5l3b1uf/ljsfz3hmge+einbzb13cbmitagsllrgugxxxqdktcgpkjipfbzmn3wnvs1od4zzosy+vb8chru/yqe9qq/rpsrqb3sipqmtneie1egg3alvur3drn+i2ztr1gv3veqbef34dx2/9mi=</latexit> <latexit sha1_base64="udqcuvokqf5ylbd81gwglcbf5b8=">aaac83icdzllbhmxfiad4vags1nyshkrizuf0uyfbmuqbfhvrsjtpuw68jhnequ+yfzmk1p+ku5qwfimvabvg5ooejnaksz9ot/vy/scl4pry9p0vy+6d//bw0c7j+mnt5893+3vvtg1styerkqyqc9lbibrasnllynzpqhzksfzeflpxc8a0izk8duufuw4nglauyjtsbx93vnhcjwnreb382bxtugp0mg6jmrbzk0yodzoir3ez3wqsc1bwmkwmemsvxbislaumpbxxhtqmfzigyydfjidmbj1zx3yjmsmssv1wmim6+zfoxzmxix5gzwc27npslxyx2xc2+rjxfghaguc3b1u1syxmlm1izlsdcsyzrcyabrumpa51pjy0kw4zgvceck5floxu9f4d+hezd5vaquvd+4it1l5cerinrzgsqobnilgzirv8irud5l3b1uf/ljsfz3hmge+einbzb13cbmitagsllrgugxxxqdktcgpkjipfbzmn3wnvs1od4zzosy+vb8chru/yqe9qq/rpsrqb3sipqmtneie1egg3alvur3drn+i2ztr1gv3veqbef34dx2/9mi=</latexit> Results x space z space g 1 (~x) [Rezende, Mohamed, & Wierstra 2014]

13 Neuroscience Application [Gao, Archer, Paninski & Cunningham 2016] ~x ~z Poiss(f (~z )) <latexit sha1_base64="4i1nrjq/7y1bym3iey4ea3zaejy=">aaadehicdzllbhmxfiad4vbclyulg4sikzugmqmq6lkcdasqskstfker43gaq76m7dptjmyv0afpdpul4gv4g5x0hegcr7l063y/j32ops6kcjcmvxrjnbv37j/yedh89pjj02et3efhzpsw8t4z0tjtmxvccs37iedy08jyqsasn4wvpi75scwte0z/gxnbh4qea5elrigmrq03pjrhr5huc0ycujgoclorfm8i50inhxgycqcdanjbg7xaatddbd4wws3aqi7ealfxk0wmkxxxwcr1bpclbqw9tscy5kfjssclyi7oor9eqanibuhxbqx8omymodc2lg14lf17h6fkubkar+fy1m6tlzp/yoms8oohf7oogwt2e1besgwgl2eej8jybniebwvwxltinqwwmoitbdaj5pfmkex1xbohq+dp/nsshhuatgjenxewryhhylbuu8i4vm4zmk9lhjzjy32fbb+/veipbf7robiq8mwnjep/timrwaqvz3i2cry1wgwcxrgaksifndo04/tnm6+9ly73u1nazt6/ax9+qh/cdnqjxqeoytb7dig+or7qi4au0dw6qd+tq+q6+zbc3fqtrr3nbvql5mdv4wocva==</latexit> p(~z ) <latexit sha1_base64="mzoui/nenodozempnwvdwer1nac=">aaac63icdzllbhmxfiad4vagw1uwbcwipligmqmqyfnbpquqskqtyqsvxzntwpvntic0sfwu3agy7nv0bxgbpokikqkcydkv8/22z/fxqtmzlst+9zj79x88fltxoh3y9nnzf5tb20dw1ybckcquzeljlhamyeiy43cidrbrcjgulz43/hggxjilv7q5hreg55jvjbixu9/0tjedihdvzzb72sbbbl4xesv6qi3ds63ebtfrtbyghexe2lgeatf2xdhgoys0qc1oqi/ioyyilesahftlxqg/izkjrpsjszq8zp69wxnh7vyu0smim9oua5l/yqpavr/hnkldo5d07qkq5tgp3lspj8wadxwebaggxvoxnrjdqiuplkafho9ucuhkxbdmzoi/9e/yefabmzp4f1o4puo8ajtdpnxkwqpnszjykviiiuf7ppjdtyp+wnx/pqdgrn50sgn3b108u4y0meflfyxasogc0lzfrzsmkxnsop+8o+11cbq7ylnb/uv9f+9t+xm20cv0gu2ghh1ae2gfhaihokigk3snfiyiuup+jnd31qtx7nmjvik5+q1/rvnk</latexit> f <latexit sha1_base64="z+vg35/w+jutrabd7h0ejs+88ma=">aaacyhicdzhnbhmxemed5assxy0cuvhesfyidiskofb0ungoiktasnk0cpzzxoq/zm+mjsxfeasu8as8ew+dk64qswaks3/n7z9jj2dipfbyfl862z279+4/2huyp3r85omz/ypn5940jkofg2ncymi8skghjwildkwdpiyslibz4xw/widzwugvulqwuuxki1pwhik1qmcvzgdzel9b9ip10f1rtqjl2jgbh3r+vlpdgwuauwted8vc4igwh4jlihnvelcmz9kvdjputiefhfwdi32dmlnag5eorrro/l0rmpj+qsbjqrjo/dzbjf/fhg3wh0zbansgah57ud1iioaupqdt4ycjxcbbubpprztpmgmc0x/leaxhmhulmj6gsuhfdjfhbrnjjkbnywixfrob0xuuhdpegg+bnqnhjusyqjr1d2umhzun/ljwv55tpxjftcqzdkfbehgt6ai4vdkg8xbmy562w27vclech/bkold+ftc9+tjuey+8jk/ig1ks9+sinjaz0iecspknfcc/sk+zza6z5a0167q1l8hgzf9/a5t+5qk=</latexit> linear dynamical system (LDS) spike trains