,19(67,*$7,1*(<(029(0(176,11$785$//$1*8$*($1'& 6285&(&2'(   E\ 3DWULFN53HDFKRFN  6XEPLWWHGLQ3DUWLDO)XOILOOPHQWRIWKH5HTXLUHPHQWV IRUWKH'HJUHHRI 0DVWHURI&RPSXWLQJDQG,QIRUPDWLRQ6\VWHPV        <281*672:167$7(81,9(56,7< 0D\       ,19(67,*$7,1*(<(029(0(176,11$785$//$1*8$*($1'& 6285&(&2'(   3DWULFN53HDFKRFN   ,KHUHE\UHOHDVHWKLVWKHVLVWRWKHSXEOLF,XQGHUVWDQGWKDWWKLVWKHVLVZLOOEHPDGH DYDLODEOHIURPWKH2KLR/,1.(7'&HQWHUDQGWKH0DDJ/LEUDU\&LUFXODWLRQ'HVNIRU SXEOLFDFFHVV,DOVRDXWKRUL]HWKH8QLYHUVLW\RURWKHULQGLYLGXDOVWRPDNHFRSLHVRIWKLV WKHVLVDVQHHGHGIRUVFKRODUO\UHVHDUFK   6LJQDWXUH         Patrick R. Peachock6WXGHQW  'DWH   $SSURYDOV       Bonita Sharif7KHVLV$GYLVRU   'DWH       Alina Lazar&RPPLWWHH0HPEHU   'DWH       John Sullins&RPPLWWHH0HPEHU   'DWH        6DO6DQGHUV$VVRFLDWH'HDQRI*UDGXDWH6WXGLHV 'DWH  LLL   $EVWUDFW ,VWKHUHDQLQKHUHQWGLIIHUHQFHLQWKHZD\SURJUDPPHUVUHDGQDWXUDOODQJXDJHWH[W FRPSDUHG WR VRXUFH FRGH" 'RHV H[SHUWLVH SOD\ D UROH LQ WKH UHDGLQJ EHKDYLRU RI SURJUDPPHUV",QRUGHUWRVWDUWDQVZHULQJWKHVHTXHVWLRQVZHFRQGXFWDFRQWUROOHG H[SHULPHQWZLWKQRYLFHDQGQRQQRYLFHSURJUDPPHUVZKLOHWKH\UHDGVPDOOVKRUWVQLSSHWV RIQDWXUDOODQJXDJHWH[WDQG&VRXUFHFRGH7KHVWXG\ZDVFRQGXFWHGZLWKVWXGHQWV UHFUXLWHGIURPDQ,QWURGXFWLRQWR3URJUDPPLQJFODVVLQWKH&RPSXWHU6FLHQFHDQG ,QIRUPDWLRQ6\VWHPVGHSDUWPHQWDW<RXQJVWRZQ6WDWH8QLYHUVLW\7KHVWXGHQWVZHUHHDFK JLYHQWHQWDVNVDVHWRIVHYHQSURJUDPVDQGWKUHHQDWXUDOODQJXDJHWH[WV7KHRUGHURIWKH WDVNV ZDV UDQGRPL]HG ZLWKLQ HDFK W\SH 7KH\ ZHUH DVNHG RQH RI WKUHH UDQGRP FRPSUHKHQVLRQTXHVWLRQVDIWHUHDFKWDVN:HXVHVHYHUDOOLQHDULW\PHWULFVWKDWZHUH SUHVHQWHGUHFHQWO\LQDVLPLODUH\HWUDFNLQJVWXG\DQGUHSRUWRQWKHILQGLQJV 7KHUHVXOWVLQGLFDWHWKDWQRYLFHVDQGQRQQRYLFHVERWKUHDGVRXUFHFRGHOHVV OLQHDUO\WKDQQDWXUDOODQJXDJHWH[W:HGLGQRWILQGDQ\GLIIHUHQFHVEHWZHHQQRYLFHVDQG QRQQRYLFHVEHWZHHQQDWXUDOODQJXDJHWH[WDQGVRXUFHFRGH:HGLVFXVVWKHLPSOLFDWLRQV RIWKLVZRUNDORQJZLWKGLUHFWLRQVIRUIXWXUHZRUN     LY  $FNQRZOHGJHPHQWV ,�G OLNH WR DFNQRZOHGJH P\ DGYLVRU 'U 6KDULI IRU SXWWLQJ XS ZLWK P\ SURFUDVWLQDWLRQDQGSXVKLQJPHWRFRQVWDQWO\JHWZRUNGRQH:LWKRXWKHUFRQVWDQW PRWLYDWLRQ,ZRXOG�YHQHYHUEHFRPHLQWHUHVWHGLQP\VWXG\7HUHVD%XVMDKQIRUKHU DVVLVWDQFHLQDQDO\VLVDQGUXQQLQJH\H&RGHDQG$KUD]+XVDLQIRUKLVVFULSWZULWLQJDELOLW\ WKDWKHOSHGLQDQDO\VLV 0\WKHVLVFRPPLWWHH'U/D]DUDQG'U6XOOLQVIRUWDNLQJWLPHRXWRIWKHLUEXV\ VFKHGXOHVWRVXSSRUWPHDQG'U6FKXHOOHUZLWKRXWKLP,ZRXOG�YHQHYHUEHHQDJUDGXDWH DVVLVWDQWDQG,ZRXOG�YHQHYHUSXUVXHGP\0DVWHU�V'HJUHH ,DOVRZDQWWRDFNQRZOHGJHWKHVWDIIRI8SWRZQ3L]]DZKRSXWXSZLWKPHKRXU DIWHUKRXUPDQ\HYHQLQJVFRQVLVWHQWO\EULQJLQJPHIRRGWRNHHSPHJRLQJDQGNHHSLQJPH PRWLYDWHGZKLOH,VDWUHDGLQJUHVHDUFKLQJDQGDQDO\]LQJ          Y  7$%/(2)&217(176  /,672)),*85(69,,, /,672)7$%/(6,;  ,1752'8&7,21  0RWLYDWLRQ  5HVHDUFK4XHVWLRQV  &RQWULEXWLRQV  2UJDQL]DWLRQ  %$&.*5281'$1'5(/$7(':25.   $Q2YHUYLHZRI(\HWUDFNLQJ7HFKQRORJ\  &RGH0DLQWHQDQFH  3URJUDP&RPSUHKHQVLRQ  (\HWUDFNLQJ6WXGLHVLQ6RIWZDUH(QJLQHHULQJ  (\HWUDFNLQJ6WXGLHVLQ&RPSXWHU(GXFDWLRQ  2WKHU%LRPHWULFEDVHG6WXGLHVRQ3URJUDP&RPSUHKHQVLRQ  'LVFXVVLRQ  7+(678'<  6WXG\2YHUYLHZ  ,QGHSHQGHQWDQG'HSHQGHQW9DULDEOHV  3DUWLFLSDQWV  YL   7DVNV  'DWD&ROOHFWLRQDQG$SSDUDWXV  6WXG\3URFHGXUHDQG,QVWUXPHQWDWLRQ  678'<5(68/76  3URFHVVLQJWKH'DWDIRU$QDO\VLV  &RPSUHKHQVLRQ6FRUHV  7LPH7DNHQ  54,VWKHUHDQLQKHUHQWGLIIHUHQFHLQWKHZD\QRYLFHSURJUDPPHUVUHDG QDWXUDOODQJXDJHWH[WFRPSDUHGWRVRXUFHFRGH"  1RQQRYLFHSURJUDPPHUV1DWXUDOODQJXDJHWH[WYVVRXUFHFRGH  54'RHVH[SHUWLVHSOD\DUROHLQWKHUHDGLQJEHKDYLRURISURJUDPPHUVLQ SDUWLFXODUZLWKUHVSHFWWROLQHDUUHDGLQJ"  3RVW4XHVWLRQQDLUH5HVXOWVDQG4XRWHV  7KUHDWVWR9DOLGLW\  &21&/86,216$1')8785(:25.  $33(1',;$678'<0$7(5,$/5(3/,&$7,213$&.$*( $6WXG\,QVWUXFWLRQV $3UH4XHVWLRQQDLUH $7KH6WLPXOL1DWXUDO7H[W 17 DQG6RXUFH&RGH 6&  $7KH&RPSUHKHQVLRQ4XHVWLRQVIRUWKH17DQG6&6WLPXOL $3RVW4XHVWLRQQDLUH  YLL  5()(5(1&(6    YLLL  /,672)),*85(6 )LJXUH7KHQXPEHURISDUWLFLSDQWVVHOIUHSRUWLQJWKHSURJUDPPLQJODQJXDJHVWKH\ NQHZRUDUHFXUUHQWO\OHDUQLQJ )LJXUH<HDUVRISURJUDPPLQJH[SHULHQFHZLWKLQSDUWLFLSDQWV )LJXUH6RXUFHFRGH6& &DOFXODWH$YHUDJH ZLWKOLQH WRS DQGZRUG EHORZ $2,V )LJXUH1DWXUDO/DQJXDJH17ZLWKOLQH WRS DQGZRUG EHORZ $2,V )LJXUH$YHUDJH&RPSUHKHQVLRQ6FRUHIRU1RYLFHVDQG1RQ1RYLFHVZLWKLQHDFKWDVN  )LJXUH$YHUDJH&RPSUHKHQVLRQ6FRUHIRU1RYLFHVDQG1RQ1RYLFHVDFURVVDOOWDVNV )LJXUH$YHUDJHWRWDOWLPHVSHQWE\QRYLFHVRQHDFKWDVN )LJXUH$YHUDJHWRWDOWLPHVSHQWE\QRQQRYLFHVRQHDFKWDVN )LJXUH$YHUDJHWRWDOWLPHVSHQWRQHDFKWDVNEHWZHHQQRYLFHVDQGQRQQRYLFHV )LJXUH$YHUDJHWRWDOWLPHVSHQWRQDOOWDVNV )LJXUH$YHUDJHOLQHDULW\PHDVXUHVIRUDOOQRYLFHV )LJXUH6DFFDGHOHQJWKIRUDOOQRYLFHVLQ17DQG6& )LJXUH(OHPHQWFRYHUDJHIRUDOOQRYLFHVLQ17DQG6& )LJXUH$YHUDJHOLQHDULW\PHDVXUHVIRUDOOQRQQRYLFHV )LJXUH6DFFDGHOHQJWKIRUDOOQRQQRYLFHVLQ17DQG6& )LJXUH(OHPHQWFRYHUDJHIRUDOOQRQQRYLFHVLQ17DQG6&    L[  /,672)7$%/(6 7DEOH2YHUYLHZRIWKH,QGHSHQGHQWDQG'HSHQGHQW9DULDEOHV 7DEOH2YHUYLHZRIVRXUFHFRGHXVHGLQWKHVWXG\7KHSURJUDPVZHUHPDUNHGDV6& WKURXJK6&IRU&7KHGLIILFXOW\ZDVVHWEDVHGRQWKHW\SHVRIFRQVWUXFWVXVHG  7DEOH2YHUYLHZRIWKHQDWXUDOODQJXDJHWH[WVXVHGLQWKHVWXG\7KHSDUDJUDSKVZHUH PDUNHGDV17WKURXJK17 7DEOH:LOFR[RQVLJQHGUDQNHGWHVWIRU17YV6&IRU1RYLFHV 7DEOH1HHGOHPDQ:XQFK5HVXOWVFRPSDULQJWKHVWRU\RUGHUIRU17DQG6&IRUQRYLFHV  7DEOH:LOFR[RQVLJQHGUDQNHGWHVWIRU17YV6&IRUQRQQRYLFHV 7DEOH0DQQ:KLWQH\UHVXOWVIRUQRYLFHVYVQRQQRYLFHVRYHUDOOWDVNV 7DEOH3RVW4XHVWLRQQDLUHRSLQLRQV 7DEOH7KRXJKW3URFHVV4XRWHV       ,1752'8&7,21 3URJUDPPLQJLVDFRPSOH[VHWRIDFWLYLWLHVWKDWLQYROYHVERWKUHDGLQJDQGZULWLQJ VRXUFH FRGH  3DUW RI SURJUDPPLQJ LQYROYHV FRPSUHKHQGLQJ ZKDW LV EHLQJ UHDG 3URJUDPPLQJDQGFRPSUHKHQVLRQDUHGLIILFXOWWRWHDVHDSDUWVLQFHWKH\DUHYHU\PXFK LQWHUWZLQHG3URJUDPPLQJODQJXDJHVGHULYHPDQ\RIWKHLUFKDUDFWHULVWLFVIURPQDWXUDO ODQJXDJHV$/DWLQEDVHGODQJXDJHLVDQ\ODQJXDJHWKDWXVHVDOSKDEHWLFDOFKDUDFWHUVVXFK DV(QJOLVKRU*HUPDQLQVWHDGRIV\PEROV/DWLQEDVHGODQJXDJHVDUHVWUXFWXUHGWREHUHDG OHIWWRULJKWDQGWRSWRERWWRPLQDV\VWHPDWLFRUGHU3URJUDPVRQWKHRWKHUKDQGDUH LQKHUHQWO\PRUHVWUXFWXUHGDQGFRPSOH[WKDQQDWXUDOODQJXDJHWH[W:KLOHPRVW/DWLQ EDVHGODQJXDJHVDUHWDXJKWWREHUHDGOHIWWRULJKWDQGWRSWRERWWRPVRXUFHFRGHRIWHQ GRHVQRWIROORZWKHVDPHVWUXFWXUHVLQFHLWLVH[HFXWHGLQDVSHFLILFVHTXHQFH JHQHUDOO\ EDVHGRQWKHFRQWUROIORZJUDSK  7KHDLPRIWKLVUHVHDUFKLVWRFRPSDUHKRZUHDGLQJRIQDWXUDOODQJXDJHVGLIIHUV IURPVRXUFHFRGH7KHEHVWZD\WRGRWKLVLVWRORRNDWSURJUDPPHUV�H\HPRYHPHQWVZKLOH UHDGLQJQDWXUDOODQJXDJHDQGVRXUFHFRGH%DUEDUD.LWFKHQKDPZDVWKHILUVWWRSURSRVH WKHDGDSWDWLRQRIHYLGHQFHEDVHGSDUDGLJPLQVRIWZDUHHQJLQHHULQJLQDQGFDOOHGLW �(YLGHQFHEDVHG6RIWZDUH(QJLQHHULQJ (%6( 7KHXVHRIHYLGHQFHEDVHGPHGLFLQHKDG EHHQDURXQGORQJEHIRUHWKHSURSRVDORI(%6(7KHPHGLFDOILHOGVDZDKXJHJURZWKEDFN LQWKH�VDQG�VZLWKWKLVSODWIRUPVKLIW%HLQJDEOHWRYLHZDQDO\]HDQGFRPSDUH GDWDKHOSHGWRQRWRQO\EHQHILWWKHPHGLFDOILHOGEXWLQFUHDVHWKHDPRXQWRISXEOLFDWLRQV    .LWFKHQKDP'\ED -RUJHQVHQ :HEHOLHYHWKDWWKHH\HWUDFNHUVXSSRUWVWKH FRQFHSWRIHYLGHQFHEDVHGVRIWZDUHHQJLQHHULQJ $QH\HWUDFNHULVDGHYLFHWKDWXVHVLQIUDUHGWHFKQRORJ\WRUHFRUGH\HPRYHPHQWV (\HWUDFNLQJLVLPSRUWDQWEHFDXVHLWJLYHVXVDFFXUDWHGDWDDVWRZKDWDXVHULVORRNLQJDW ZKHQUHDGLQJRQDVFUHHQ,WLVGLIIHUHQWIURPILOOLQJRXWDTXHVWLRQQDLUHDIWHU\RXDUHGRQH DQDO\]LQJWKHWDVNEHFDXVHVRPHWLPHVDSHUVRQPLJKWPLVUHSRUWRQWKHDQVZHURUKDYHWKH FRUUHFWWKRXJKWSURFHVVEXWDQVZHULQFRUUHFWO\:LWKH\HWUDFNLQJZHFDQVHHWKHWKRXJKW SURFHVVHVDVWKHSHUVRQLVORRNLQJDWDQGVROYLQJWKHSUREOHPRQWKHVFUHHQ.QRZLQJKRZ DXVHUUHDGVLVFULWLFDOLQWKHHYROXWLRQRIWRROVDQGWHFKQLTXHVWRKHOSXVHUVUHDGDQG FRPSUHKHQGERWKFRGHDQGQDWXUDOODQJXDJHWH[W(\HWUDFNLQJKDVEHHQDURXQGLQWKHILHOG RIVRIWZDUHHQJLQHHULQJVLQFHWKHHDUO\�VEXWKDVEHHQJDLQLQJSRSXODULW\DWDUDSLG SDFH (\H WUDFNLQJ KDV PDQ\ SXUSRVHV WHVWLQJ YLVLRQ UHDGLQJ FRPSUHKHQVLRQ VLPXODWLRQVDQGHYHQVRIWZDUHHQJLQHHULQJ(\HWUDFNLQJKDVEHHQXVHGLQWKHVRIWZDUH HQJLQHHULQJILHOGDVHDUO\DVWKHODWH�V2QHRIWKHILUVWDQGPRVWZHOONQRZQXVHV EHLQJ0DUWKD&URVE\�VGLVVHUWDWLRQ 0&URVE\ LQWLWOHG�1DWXUDO9HUVXV &RPSXWHU/DQJXDJHV$5HDGLQJ&RPSDULVRQ�6LQFHWKHUHOHDVHRIWKLVGLVVHUWDWLRQ WKHUHKDVEHHQDJURZLQJLQWHUHVWLQH\HWUDFNLQJDQGVRIWZDUHHQJLQHHULQJ6WXGLHVKDYH EHHQGRQHLQPDQ\ILHOGVLQFOXGLQJSURJUDPFRPSUHKHQVLRQ %URRNV%XVMDKQ  6FKXOWHD6KDULI 0DOHWLF FRPSXWHUHGXFDWLRQ %XVMDKQHWDO  GHEXJJLQJ %HGQDULN 7XNLDLQHQ VRIWZDUHWUDFHDELOLW\ $OL6KDUDIL*XpKpQHXF    $QWRQLRO6KDULI .DJGL:DOWHUV)DOFRQH6KLEEOH 6KDULI :DOWHUV6KDIIHU6KDULI .DJGL VRIWZDUHYLVXDOL]DWLRQ -HWW\ DQGXVDELOLW\ ,QWKLVWKHVLVZHFRQGXFWDVWXG\WRWHVWKRZVWXGHQWVFRPSUHKHQG&FRGHDQG QDWXUDOODQJXDJHWH[WV:HDQDO\]HGWKUHHVKRUW(QJOLVKWH[WVDQGVHYHQVKRUW&FRGH VQLSSHWVZLWKVHYHUDOVWXGHQWVKDYLQJDYDULHGVHWRIH[SHUWLVH7KHJRDOLVWRGHWHUPLQHLI WKHUHDUHGLIIHUHQWVW\OHVRIUHDGLQJSDWWHUQVEHWZHHQGLIIHUHQWFDWHJRULHVRIXVHUV,Q SDUWLFXODUZHDUHLQWHUHVWHGLQFKDUDFWHUL]LQJOLQHDUUHDGLQJ2XUSDUWLFLSDQWVZHUHUHFUXLWHG IURPLQWURGXFWRU\DVZHOODVDGYDQFHGFODVVHVLQWKH&RPSXWHU6FLHQFHDQG,QIRUPDWLRQ 6\VWHPVGHSDUWPHQWDW<RXQJVWRZQ6WDWH8QLYHUVLW\  0RWLYDWLRQ 7KHUHKDVEHHQDUDSLGO\JURZLQJLQWHUHVWLQH\HWUDFNLQJVWXGLHV'XULQJWKH�V WKHUHZHUHOHVVWKDQSDSHUVSXEOLVKHG:KLOHVWXGLHVKDYHJLYHQLQVLJKWVDVWRKRZXVHUV UHDGFRGHDQGQDWXUDOODQJXDJHWH[WVHSDUDWHO\WKHUHKDVQ�WEHHQPXFKSURJUHVVLQWKH FRPSDULQJRIQRYLFHVDQGQRQQRYLFHVUHDGLQJFRGH7KHUHDOVRKDV\HWWREHSURRILIWKH H[SHUWLVHOHYHORIDSURJUDPPHUGHWHUPLQHVKRZWKH\UHDGDQGFRPSUHKHQGFRGH7KH DELOLW\WRUHDGFRGHLVMXVWDVLPSRUWDQWDVWKHDELOLW\WRZULWHFRGH7KHUHLVDJURZLQJ QXPEHURISURJUDPVWKDWDUHEHLQJZRUNHGRQWKDWKDYHEHHQEXLOWXSRQSUHYLRXV SURJUDPPLQJNQRZLQJZKDWDSURJUDPGRHVSULRUWRXVLQJLWDVDEDVHEXLOGLQJEORFNLV FULWLFDOWRWKHVXFFHVVRIWKHEXLOWSURJUDP:LWKDEHWWHUXQGHUVWDQGLQJRIKRZDQRYLFH YVDQH[SHUWUHDGVDQGFRPSUHKHQGVFRGHWRROVFDQEHGHYHORSHGWRWHDFKSHRSOHWREH EHWWHUSURJUDPPHUV7KHUHLVQRSXEOLVKHGUHVHDUFKWKDWTXDQWLILHVWKHUHDGLQJDSSURDFKRI    QDWXUDOODQJXDJHWH[WDQGVRXUFHFRGH EHVLGHV %XVMDKQHWDO IRU-DYD 7KLVVWXG\ VHHNVWREULGJHWKDWJDS  5HVHDUFK4XHVWLRQV 7KHIROORZLQJDUHWKHUHVHDUFKTXHVWLRQVZHVHHNWRDQVZHU x 54 ,V WKHUH DQ LQKHUHQW GLIIHUHQFH LQ WKH ZD\ QRYLFH DQG QRQQRYLFH SURJUDPPHUVUHDGQDWXUDOODQJXDJHWH[WFRPSDUHGWRVRXUFHFRGH" x 54 'RHVH[SHUWLVHSOD\DUROHLQWKHUHDGLQJEHKDYLRURISURJUDPPHUVLQ SDUWLFXODUZLWKUHVSHFWWROLQHDUUHDGLQJ" %HWWHUXQGHUVWDQGLQJRIWKHGLIIHUHQWUHDGLQJSDWWHUQVRIQRYLFHDQGQRQQRYLFH SURJUDPPHUVZLOOKHOSXVWRDGYDQFHWHDFKLQJVW\OHVWRROVDQGDOVRLPSURYHFRGLQJ SUDFWLFHV7KLVVWXG\LVRQO\DILUVWVWHSWRZDUGVWKLVORQJWHUPJRDO 54EDVLFDOO\VHHNVWRGHWHUPLQHLIWKHOLQHDUUHDGLQJSDWWHUQVIRXQGLQQDWXUDO ODQJXDJHWH[WWUDQVIHUVWRVRXUFHFRGH54VHHNVWRVHSDUDWHWKHH[SHUWLVHDQGFRPSDUHLI WKHUHDUHGLIIHUHQFHVLQWZRVXEJURXSVLHQRYLFHVYVQRQQRYLFHV  &RQWULEXWLRQV 7KHPDLQFRQWULEXWLRQRIWKLVWKHVLVLVDVWXG\RQWKHH\HPRYHPHQWVLQQDWXUDO ODQJXDJHWH[WDQG&VRXUFHFRGHZLWKLQQRYLFHDQGQRQQRYLFHSURJUDPPHUV7KHVWXG\ FRQVLVWVRIWHVWSDUWLFLSDQWVUHDGLQJVRXUFHFRGHDVZHOODVQDWXUDOWH[WDQGDQVZHULQJD FRPSUHKHQVLRQTXHVWLRQDIWHUHDFK%DVHGRQWKHH\HPRYHPHQWVDVHWRIOLQHDULW\ PHDVXUHVZHUHFDOFXODWHGEDVHGRQUHFHQWSULRUZRUN7KHOLQHDULW\PHDVXUHVDUHXVHGWR TXDQWLI\OLQHDUUHDGLQJLQQDWXUDOODQJXDJHWH[WDQGVRXUFHFRGH7KLVVWXG\LVDSDUWLDO    UHSOLFDWLRQRIWKHUHFHQWVWXG\SXEOLVKHGDWWKH,QWHUQDWLRQDO&RQIHUHQFHRI3URJUDP &RPSUHKHQVLRQ ,&3& E\%XVMDKQHWDO %XVMDKQHWDO  7KHGLIIHUHQFHVDUHEULHIO\KLJKOLJKWHGQH[W2XUVWXG\XVHGDGLIIHUHQWH[SHULPHQWDO GHVLJQDQGZDVSDUWRID&VHPHVWHUORQJFODVVDOWKRXJKRWKHUVWXGHQWVIURPVHQLRU&6 FODVVHVZHUHDOVRUHFUXLWHG%XVMDKQHWDO�VVWXG\ZDVGRQHDIWHUVWXGHQWVFRPSOHWHGDQ RQOLQHPRGXOHLQ-DYD8QOLNH%XVMDKQ�VVWXG\RXUVWXG\GLGQRWVWXG\SURIHVVLRQDO SURJUDPPHUVUDWKHURQO\QRYLFHVDQGQRQQRYLFHV:HUHIHUWRDQ\SDUWLFLSDQWDVDQRQ QRYLFHLIWKH\ZHUHUHFUXLWHGIURPDFODVVWKDWZDVDWDKLJKHUVWDQGLQJWKDQWKH,QWURGXFWLRQ WR&RPSXWHU3URJUDPPLQJFODVV WKDWUHTXLUHVQRSULRUEDFNJURXQGLQSURJUDPPLQJ  6LQFHWKHVHQRQQRYLFHVDUHQRWH[SHUWVZHSUHIHUQRWWRXVHWKHWHUPH[SHUWVWRUHIHUWR WKHP  2UJDQL]DWLRQ 7KLVWKHVLVLVRUJDQL]HGDVIROORZV7KHQH[WFKDSWHUJLYHVDEULHILQWURGXFWLRQWR H\HWUDFNLQJDQGUHODWHGZRUN&KDSWHUSUHVHQWVWKHGHWDLOVRIWKHH[SHULPHQWDOVHWXSIRU WKHVWXG\&KDSWHUGLVFXVVHVREVHUYDWLRQVDQGUHVXOWV&KDSWHUFRQFOXGHVWKHWKHVLVDQG SUHVHQWVIXWXUHZRUN      %$&.*5281'$1'5(/$7(':25.  7KLVFKDSWHUWDONVDERXWDQRYHUYLHZRISURJUDPFRPSUHKHQVLRQDKLVWRU\RIH\H WUDFNLQJVWXGLHVDQGDJHQHUDOGLVFXVVLRQRQH\HWUDFNLQJKLVW RU\,WKDVRQO\EHHQZLWKLQ WKHSDVWIHZ\HDUVWKDWH\HWUDFNLQJLQVRIWZDUHHQJLQHHULQJKD VEHFRPHDJURZLQJDUHDRI VWXG\(\HWUDFNLQJLQVRIWZDUHHQJLQHHULQJKDVSULPDULO\EHHQ EDVHGDURXQGDIHZIRFL PRGHOFRPSUHKHQVLRQFRGHFRPSUHKHQVLRQDQGGHEXJJLQJ%HFDXV HWKLVVWXG\IRFXVHVRQ FRGHUHDGLQJDQGFRPSUHKHQVLRQRIVDLGFRGHWKHFKDSWHUZLOOF RYHUDEULHIKLVWRU\RIERWK DVZHOODVDVPDOORYHUYLHZRIH\HWUDFNHUVDQGH\HWUDFNLQJVWXGLHV  $Q2YHUYLHZRI(\HWUDFNLQJ7HFKQRORJ\ $QH\HWUDFNHULVDGHYLFHWKDWLVDEOHWRGHWHFWZKHUHDXVHU LVORRNLQJRQWKHVFUHHQ 7KHVFUHHQXVXDOO\LVGLVSOD\LQJDVWLPXOLRILQWHUHVW LQRXUFDVHVRXUFHFRGHRUQDWXUDO WH[W :KHQXVLQJDQH\HWUDFNHUWKHUHDUHWZRW\SHVRIGDWDWKDWDUHW\SLFDOO\JHQHUDWHG H\HIL[DWLRQVDQGVDFFDGHV$QH\HIL[DWLRQLVDUHVWLQJRIWKHH\HRQSDUWRIWKHVWLPXOL IRUDVHWDPRXQWRIWLPH$VDFFDGHLVDUDSLGPRYHPHQWIURPRQHH\HIL[DWLRQWRWKH QH[W7KLVDPRXQWRIWLPHFDQYDU\VOLJKWO\EXWLVRIWHQEHWZHHQWRPLOOLVHFRQGV )L[DWLRQVDQGVDFFDGHVDUHFRQQHFWHGE\VFDQSDWKV6RPHDQDO\VLVWRROVWHOOWKHGLIIHUHQW GXUDWLRQVRIIL[DWLRQVE\PDUNLQJDGRWRQWKHVWLPXOLWKDWJURZVWKHORQJHUWKHH\HVUHVW LQRQHVSRWIL[DWLRQGDWDFDQDOVREHUHFRUGHGDVDPHWULFRIWLPH,QH\HWUDFNLQJVWXGLHV WKHSURFHVVRILQIRUPDWLRQKDSSHQVGXULQJIL[DWLRQVEXWQRWGXULQJVDFFDGHV 5D\QHU &KDFH6ODWWHU\ $VKE\     (\HWUDFNHUVFRPHLQWZRGLIIHUHQWIRUPVLQWUXVLYHDQGQRQLQWUXVLYH$QRQ LQWUXVLYHH\HWUDFNHULVQRWZRUQE\WKHXVHULQDQ\ZD\7KHRULJLQDOQRQLQWUXVLYHH\H WUDFNHUVILUVWIXQFWLRQHGE\UHIOHFWLQJOLJKWRIIRIDXVHU�VSXSLOV6RPHQRQLQWUXVLYHH\H WUDFNHUVDUHYLGHREDVHGDQGXVHFDPHUDVWRGHWHUPLQHH\HPRYHPHQWVE\RWKHUREMHFWVRQ WKHIDFHVXFKDVWKHH\HEURZVDQGQRVH7KHVHW\SHVRIH\HWUDFNHUVXVHLQIUDUHGWRVHHD UHIOHFWLRQRIIRIWKHFRUQHD  7KHVHFRQGIRUPRIH\HWUDFNHULVFRQVLGHUHGDQLQWUXVLYHH\HWUDFNHU7KHVHW\SHV RIH\HWUDFNHUVDUHZRUQE\WKHXVHU7KHVHFDQFRQVLVWRIDKHDGEDQGZLWKFDPHUDV JODVVHVRUJRJJOHV7KHPRVWQRWLFHDEOHSUREOHPZLWKWKHVHW\SHVRIH\HWUDFNHUVLVWKDW XVHUVDUHXQFRPIRUWDEOHZHDULQJWKHP7KHLQWUXVLYHH\HWUDFNHUVDUHJHQHUDOO\PRUH DFFXUDWHWKDQWKHYLGHREDVHGRQHV)RURXUSXUSRVHVWKHYLGHREDVHGH\HWUDFNHUZRUNV ZHOO  &RGH0DLQWHQDQFH $ODUJHSDUWRIKDQGOLQJFRPSXWHUSURJUDPPLQJLVWKHPDLQWHQDQFHDQGXSNHHSRI FRGH&RGHPDLQWHQDQFHLVGHILQHGDVWKHWRWDOLW\RIDFWLYLWLHVUHTXLUHGWRSURYLGHFRVW HIIHFWLYHVXSSRUWWRDVRIWZDUHV\VWHP$FWLYLWLHVDUHSHUIRUPHGGXULQJWKHSUHGHOLYHU\ VWDJHDVZHOODVWKHSRVWGHOLYHU\VWDJH /LHQW]6ZDQVRQ 7RPSNLQV ,WLV HVWLPDWHGWKDWH[SHQGLWXUHVDVKLJKDVRIDQ,QIRUPDWLRQ6\VWHPVEXGJHWFDQEH VSHQW RQ VRIWZDUH PDLQWHQDQFH  %DQNHU 'DYLV  6ODXJKWHU    3URJUDP PDLQWHQDQFHFDQRIWHQEHGLIILFXOWIRUDSURJUDPPHUEHFDXVHWKHFRGHZDVZULWWHQE\ DQRWKHUSURJUDPPHU$SURJUDPPHUFDQEHPRUHSUHSDUHGIRUFRGHPDLQWHQDQFHWDVNVLI WKH\DUHDOUHDG\DEOHWRFRPSUHKHQGFRGHRQDKLJKOHYHO,QUHVHDUFKIRFXVHGRQ    SURJUDPFRPSUHKHQVLRQKDGHVWLPDWHGPRUHWKDQSURIHVVLRQDOSURJUDPPLQJWLPHZDV VSHQWRQFRGHPDLQWHQDQFHDQGXSNHHSRIDOUHDG\ZULWWHQSURJUDPV 3HQQLQJWRQ   3URJUDP&RPSUHKHQVLRQ &RPSUHKHQVLRQLVGHILQHGE\0HUULDP:HEVWHUDVWKHDFWRUDFWLRQRIJUDVSLQJ ZLWKWKHLQWHOOHFW$SURJUDPRUFRPSXWHUSURJUDPLVGHILQHGDVDVHTXHQFHRILQVWUXFWLRQV ZULWWHQWRSHUIRUPDVSHFLILHGWDVNZLWKDFRPSXWHU3URJUDPFRPSUHKHQVLRQLVWKHUHDGLQJ DQGXQGHUVWDQGLQJRIWKHOLQHVRIFRGHZLWKLQDFRPSXWHUSURJUDP 3URJUDPFRPSUHKHQVLRQLVDQLPSRUWDQWSDUWRIFRPSXWHUSURJUDPPLQJ6RIWZDUH PDLQWHQDQFHLVKHDYLO\UHOLDQWRQDSURJUDPPHU�VFRPSUHKHQVLRQRIWKHSURJUDPWKDWLV EHLQJPDLQWDLQHG7RKHOSLPSURYHWKHDELOLW\WRFRPSUHKHQGDFRPSXWHUSURJUDPZH PXVWILUVWXQGHUVWDQGKRZDSURJUDPLVZULWWHQDQGDVVHPEOHG3UDFWLFDOO\DQHQRUPRXV DPRXQWRIWLPHLVVSHQWGHYHORSLQJSURJUDPVDQGHYHQPRUHWLPHLVVSHQWGHEXJJLQJWKHP DQGVRLIZHFDQLGHQWLI\IDFWRUVWKDWH[SHGLWHWKHVHDFWLYLWLHVDODUJHDPRXQWRIWLPHDQG PRQH\FDQEHVDYHG 0(+DQVHQ*ROGVWRQH /XPVGDLQH )LQGLQJWKRVH H[SHGLWLQJIDFWRUVFDQEHGRQHE\VWXG\LQJWKHH\HPRYHPHQWVRISURJUDPPHUV %URRNVILUVWGLVFXVVHGWKHXVHRIEHDFRQVRURUWKRGR[VHJPHQWVRIFRGH %URRNV  5XYHQ%URRNVSURGXFHGDWKHRU\RIFRPSUHKHQVLRQRQFRPSXWHUSURJUDPV7KH UHVHDUFKGLVFXVVHGWKHRULHVDQGHYLGHQFHLQYROYLQJSURJUDPFRPSUHKHQVLRQDQGGLIIHUHQW K\SRWKHVHVEHKLQGSURJUDPV %HDFRQVKDYHVLQFHEHHQVWXGLHGWRVKRZWKDWWKH\HQFRPSDVVPRUHWKDQMXVWFRGH IUDJPHQWV *HOOHQEHFN &RRN 6LQFHWKHIRXQGLQJGLVFXVVLRQVRIEHDFRQVWKH\ DUHVWLOOQRWNQRZQWREHDPDMRUSDUWLQSURJUDPFRPSUHKHQVLRQRUWREHH[SODQDWLRQRID    SURIHVVLRQDOSURJUDPPHUV�XQGHUVWDQGLQJDQGFRPSUHKHQVLRQRIFRGH&URVE\HWDOLQ SURYLGHGDFRPSOHWHUHVHDUFKDUWLFOHGLVFXVVLQJDQGVWXG\LQJEHDFRQVDQGVWDWHG EHDFRQVPD\EHLQWKHH\HRIWKHEHKROGHU 0(&URVE\6FKROW] :LHGHQEHFN  )DQQRWHGVLPLODUUHVXOWVRQDVWXG\GRQHLQDIILUPLQJUHVXOWVWKDWVXJJHVWEHDFRQ LGHQWLILFDWLRQLVLQWKHH\HVRIWKHEHKROGHUEXWQRWLFHGWKDWWKHSUHVHQFHRIFRPPHQWVGRHV KDYHDQHIIHFWRQFRGHUHDGLQJ )DQ  ,Q6RORZD\HWDOGHVLJQHGWZRVWXGLHV 6RORZD\/DPSHUW/HWRYVN\ /LWWPDQ 3LQWR WRILQGLPSRUWDQWFRQFHSWXDOUHODWLRQVRIFRGH7KLVZDVGRQHE\ WDNLQJDORRNDWWKHGLIIHUHQWIRFXVHVRQVRIWZDUHGRFXPHQWDWLRQ7KHJRDOZDVWRILQG GLIIHUHQWUHSUHVHQWDWLRQVWKDWFRXOGEHQHILWDSURJUDPPHUZKHQPDNLQJFKDQJHVWRWKDW FRGH7KHILUVWVWXG\GHDOWZLWKH[SORULQJWKHGHVLJQRIVRIWZDUHGRFXPHQWDWLRQIRU PDLQWHQDQFH,WIRFXVHGRQGLIIHUHQWW\SHVRIUHSUHVHQWDWLRQVWKDWFRXOGSRVVLEO\KHOS SURJUDPPHUVZKHQWKH\PDNHFKDQJHVWRDSURJUDP7KHVHFRQGVWXG\GHDOWZLWK H[SORULQJWKHW\SHVRILQIRUPDWLRQWKDWDUHWUDGHGEDFNDQGIRUWKDPRQJWKHSDUWLFLSDQWVLQ IRUPDOGHVLJQDQGFRGHLQVSHFWLRQ 7KHILUVWVWXG\ZDVWRWDNHDORRNDWWKHGLIIHUHQWPLQGVHWVORJLFDQGVWDWLF NQRZOHGJH WKDW SURIHVVLRQDO SURJUDPPHUV XVH  7R GR WKLV WKH\ JDYH GLIIHUHQW SURJUDPPHUVDSURJUDPWKDWPDQDJHGDVPDOOGDWDEDVHDVZHOODVWKHGRFXPHQWDWLRQDQG FRGHRIWKHSURJUDP*LYHQWKLVLQIRUPDWLRQWKHSURJUDPPHUVZHUHJLYHQDWDVNWRUHVWRUH DGHOHWHGUHFRUGZLWKLQWKHVDPHVHVVLRQRILWVGHOHWLRQ2EVHUYDWLRQZDVIRXQGWKDW SURIHVVLRQDOOHYHOSURJUDPPHUVXVHGWZRGLIIHUHQWW\SHVRIVWUDWHJLHVUHODWHGWRHDFKWKH FRGHDQGWKHGRFXPHQWDWLRQ$IWHUREVHUYLQJDQGFUHDWLQJDQHQKDQFHGVWUDWHJ\IRUUHDGLQJ    DQGSHUIRUPLQJDWDVNLWZDVGHWHUPLQHGWKDWPRUHRSWLPL]DWLRQFRXOGEHGRQHWRKHOS HQKDQFHWKHVWUDWHJ\ 7KHVHFRQGVWXG\WRRNDORRNDWLQIRUPDWLRQWKDWZDVVKDUHGEHWZHHQSURJUDPPHUV LQDIRUPDOUHYLHZSURFHVV$IWHUWDNLQJWKHWLPHWRPRQLWRUDIRUPDOUHYLHZWHDPLWZDV VKRZQWKDWOHVVWKDQKDOIRIWKDWWLPHZDVIRFXVHGRQWKHFRGHLWVHOIZKLOHPRUHWKDQKDOI ZDVIRFXVHGRQWKHFRGHEHLQJXQGHUVWDQGDEOHIRUIXWXUHSURJUDPPHUV7KHH[DPSOH PHQWLRQHGGLVFXVVHGWKDWE\DYRLGLQJVRPHRSWLPL]DWLRQRIIXQFWLRQDOLW\WLPHVSHQWRQ FRGHPDLQWHQDQFHGRZQWKHURDGFRXOGEHUHGXFHG,WZDVGHWHUPLQHGWKDWZKLOH GHORFDOL]HGSODQVZHUHLPSRUWDQWLQERWKVWXGLHVWKDWWKHQDWXUHRIWKHVHVWUDWHJLHVDUHLQ WKHH\HRIWKHSURJUDPPHUGRLQJWKHFRPSUHKHQVLRQ 7RKHOSIXUWKHUXQGHUVWDQGZKDWKDSSHQVLQWKHXSNHHSRIDSURJUDPUHVHDUFKZDV FRQGXFWHGRQFRGHQDPLQJ1DPLQJLQSURJUDPPLQJLVWKHDFWRIDVVLJQLQJDQLGHQWLILHU WRDFRQVWUXFW:KLOHVLPSOHVRXQGLQJQDPLQJGRHVQ�WKDYHDVWDQGDUGDQGLVWKHUHIRUHVHW E\WKHSURJUDPPHU$VLPSOHJXLGHOLQHWRQDPLQJLVWRXVHDQDPHWKDWLVPHDQLQJIXORU VRPHKRZUHODWHGWRWKHFRQVWUXFWLQXVH7ZRW\SHVRIQDPLQJFRQYHQWLRQVDUHFRPPRQ DPRQJSURJUDPPHUVZKHQLWFRPHVWRQDPLQJZLWKPXOWLSOHZRUGVXQGHUVFRUHVDQGFDPHO FDVH6LQFHVHYHUDOSURJUDPPLQJODQJXDJHVDUHSLFN\DERXWXVLQJDVSDFHDQXQGHUVFRUH FDQEHXVHGWRFRQQHFWDQDPHZLWKPXOWLSOHZRUGV7KHVHFRQGFRQYHQWLRQFDPHOFDVH FDSLWDOL]HVRQO\WKHILUVWOHWWHURIHDFKFRQQHFWHGZRUG &RPPRQQDPLQJFDQDOVRWDNHLWVIRUPIURPQDWXUDOODQJXDJH6LQFHSURJUDPPLQJ LVFRPSULVHGRIDORWRIQDWXUDOWH[WEXWZLWKVSHFLILFNH\ZRUGVWKLVLVVRPHWKLQJWKDW RFFXUVQDWXUDOO\/RRNLQJDWWKH-DYDSURJUDPPLQJODQJXDJHWKHUHDUHGLIIHUHQWPHWKRGV    ZLWKLQ WKH -DYD FODVV MDYDXWLO9HFWRU WKDW XWLOL]H FRPPRQ JUDPPDU VXFK DV UHPRYH(OHPHQW$WVHW(OHPHQW$VDQGVHW6L]H1DPLQJFDQDOVRFRQWDLQZKDWLVUHIHUUHG WRDVRYHUORDGLQJ7KLVPHDQVWKDWRQHQDPHFDQEHFRPPRQEHWZHHQVHYHUDOIXQFWLRQV :KLOHWKLVFDQEHEHQHILFLDOGXHWRGHFUHDVLQJWKHOLQHVRIFRGHRUKDYLQJOHVVQDPHVWR UHFDOOLWFDQDOVREHFRPSOLFDWHG0DQ\FRQVWUXFWVZLWKWKHVDPHQDPHFDQEHGLIILFXOWWR UHDG DQG FRPSUHKHQG IRU QRYLFH SURJUDPPHUV RU VRPHRQH XQIDPLOLDU ZLWK WKH SURJUDPPHU�VFRGLQJVW\OHV 1DPLQJLVDQLPSRUWDQWSDUWRIDOOSURJUDPV7KHUHVHDUFKH[SODLQHGVHYHUDO GLIIHUHQWZD\VWKDWSURJUDPPHUVXVHQDPLQJ3UDFWLFDOQDPLQJFRQYHQWLRQVKDYHKHOSHG WRPDNHPRVWSURJUDPPHUVPRUHHIILFLHQWLQQDPLQJVFKHPHV1DPLQJFRPELQHVERWK QDWXUDOODQJXDJHDQGDPHWKRGLFDOWKRXJKWSURFHVVDQGZKLOHLWVHHPVVLPSOHRQWKH VXUIDFHWKHUHLVDIXUWKHUFRJQLWLYHOD\HUWKDWQHHGVWREHORRNHGLQWR /LEOLW%HJHO  6ZHHWVHU  7KHGLIILFXOW\LQXQGHUVWDQGLQJDQGFRPSUHKHQGLQJRISURJUDPPLQJDUHQRWMXVW LVRODWHGLQFLGHQFHV$PXOWLQDWLRQDOVWXG\ZDVFRQGXFWHGLQE\WKH�0F&UDFNHQ JURXS�WKDWSURYHGWKDWSURJUDPPLQJLVQRHDV\WDVNWROHDUQ,QDJURXSVHWRXWWR ILQGRXWZKDWH[DFWO\WKHURRWRIWKHSUREOHPLVZKHQOHDUQLQJKRZWRSURJUDP7KH\VHW RXW ZLWK PXOWLSOH FKRLFH TXHVWLRQV WR GHWHUPLQH D SURSHU VFRUHZLWKRXW VXEMHFWLYH MXGJHPHQWDQGDOVREHFDXVHWKLVZRXOGUHPRYHDQ\ODQJXDJHIOXHQF\LVVXHVOHDYLQJDOORI WKRVHZKRPZHUHWHVWHGRQDQHYHQSOD\LQJILHOG  :KLOHWKHUHZHUHGLIIHUHQFHVEHWZHHQLQVWLWXWLRQVVXFKDVSURJUDPPLQJODQJXDJH VWXGHQWH[SHULHQFHDQGVWXGHQWPRWLYDWLRQ LPSDFWRQJUDGHVYVYROXQWHHUV WKHUHOLDELOLW\    RIWKHTXHVWLRQVWKHPVHOYHVGLGIDOOEHORZDRQWKH&URQEDFK�VFRHIILFLHQWDOSKDVFDOH PHDQLQJWKHZKROHVHWRITXHVWLRQVZDVRQO\VOLJKWO\XQUHOLDEOHEXWQRWZRUWK\RI GLVPLVVLQJDFURVVDOOLQVWLWXWLRQV7KHVWXG\JRHVLQWRGHWDLOH[SODLQLQJGRRGOLQJQRWHV DQGUHDGLQJYHUVXVZULWLQJVRXUFHFRGH:LWKDOORIWKHGDWDSUHVHQWHGWKHFRQFOXVLRQRI SURJUDPPLQJVNLOOUHVXOWVLQVWXGHQWVODFNLQJDSUHFXUVRUWRSUREOHPVROYLQJDQGWLHV GLUHFWO\WRDVWXGHQW�VDELOLW\WRUHDGFRGHUDWKHUWKDQZULWHFRGH7KHVWXG\ZKLOHEXLOW XSRQ7KH0F&UDNHQJURXS�VUHVHDUFKOHIWRSHQURRPIRUPRUHVWXGLHVLQWHUODFLQJERWK JURXSVUHVHDUFKGHVLJQVUHDGLQJFRGHDQGZULWLQJFRGH /LVWHUHWDO  0LFKDHO+DQVHQZLWK5REHUW*ROGVWRQHDQG$QGUHZ/XPVGDLQHSXEOLVKHG�:KDW 0DNHV&RGH+DUGWR8QGHUVWDQG"�7KHUHVHDUFKFRQGXFWHGXVHGSURJUDPPHUVRIDOO UDQJHVDQGZDVGRQHWRKHOSGHWHUPLQHZKDWIDFWRUVFRXOGLPSDFWWKHFRPSUHKHQVLRQ GLIILFXOW\RIWKHFRGH7KHLUUHVXOWVOHGWRDEHWWHUXQGHUVWDQGLQJWKDWH[SHULHQFHKHOSVZKHQ SURJUDPPHUVEHOLHYHWKDWWKHUHPD\EHHUURUVEXWFDQDFWXDOO\KLQGHUWKHLUDELOLW\ZKHQ WKH\KDYHQ�WEHHQWUDLQHGIRUVSHFLILFFDVHV7KHLUUHVXOWVDOVRVKRZHGWKDWYHUWLFDO ZKLWHVSDFHZDVDIDFWRULQJURXSLQJVHFWLRQVRIUHODWHGVWDWHPHQWVWRJHWKHU 0(+DQVHQ HWDO    (\HWUDFNLQJ6WXGLHVLQ6RIWZDUH(QJLQHHULQJ 6WXGLHVLQYROYLQJH\HWUDFNLQJDQGSURJUDPPLQJKDYHEHHQDVORZO\JURZLQJDUHD 6WXGLHVLQYROYLQJH\HWUDFNLQJDQGVRIWZDUHHQJLQHHULQJKDYHEHHQIHZFRPSDUHGWRRWKHU ILHOGV3URJUDPFRPSUHKHQVLRQUHVHDUFKKDVEHHQDQHYHQVPDOOHUDUHDRIH\HWUDFNLQJ 7KHILUVWSDSHUZDVSXEOLVKHGLQE\&URVE\HWDORQH\HWUDFNLQJLQVRXUFHFRGH7KH QH[WSDSHUZDVQRWSXEOLVKHGXQWLODQGWKHQQRWDJDLQXQWLO7KHILHOGKDV    TXLFNO\JURZQVLQFHWKHQWKRXJKWRLQFOXGHDSSUR[LPDWHO\SDSHUVLQWKHVRIWZDUH HQJLQHHULQJILHOGVLQFH7KHUHDVRQPRVWOLNHO\EHLQJWKDWWKHJURZLQJDFFHVVLELOLW\ WRH\HWUDFNHUVZLWKLQWKHVRIWZDUHHQJLQHHULQJILHOGLVZKDWLVFDXVLQJWKHLQFUHDVHLQ UHVHDUFK )ROORZLQJ %URRNV %URRNV   0DUWKD &URVE\ UHOHDVHG KHU GLVVHUWDWLRQ �1DWXUDO9HUVXV&RPSXWHU/DQJXDJHV$5HDGLQJ&RPSDULVRQ�LQ 0&URVE\    7KH VWXG\ IROORZHG QRQH[SHUW SURJUDPPHUV DQG WKHLU DELOLW\ WR UHDG DQG FRPSUHKHQG FRPSXWHU SURJUDPV  7KH H[SHULPHQWV OHG WR D EHOLHI WKDW FRPSXWHU SURJUDPPLQJODQJXDJHLVPRUHDGYDQFHGWKDQQDWXUDOODQJXDJHZKHQLWFRPHVWRQRQ H[SHUWVDQGWKHLUDELOLW\WRFRPSUHKHQGSURJUDPPLQJFRGH&URVE\GLGQRWLQFOXGHH[SHUW OHYHOSURJUDPPHUVWKRXJKZKLFKOHIWWKHGRRURSHQIRUUHVHDUFKWKDWZRXOGLQFOXGHH[SHUW WLHUHGSURJUDPPHUV ,Q0DUWKD&URVE\-HDQ6FKROW]DQG6XVDQ:LGHQEHFNUHOHDVHGDVWXG\WLWOHG �7KH5ROHV%HDFRQV3OD\LQ&RPSUHKHQVLRQIRU1RYLFHDQG([SHUW3URJUDPPHUV� 0( &URVE\HWDO 7KHVWXG\LQYHVWLJDWHGNH\IHDWXUHVRISURJUDPVWKDWDVVLVWLQ FRPSUHKHQVLELOLW\7KHVWXG\ZDVGRQHXVLQJSURJUDPPHUVRIDOOUDQJHVIURPQRQ SURJUDPPHUVWRDGYDQFHGH[SHUWV7KHFRQFOXVLRQVOHDGWRDEHOLHIWKDWWKRVHIHDWXUHVRU EHDFRQVDUHQRWQHFHVVDULO\WKHVDPHIRUDOOSURJUDPPHUV:KDWRQHSURJUDPPHUVHHVDV DEHDFRQPD\EHORRNHGRYHUTXLFNO\E\DQRWKHUSURJUDPPHU7KHUHVHDUFKRQFHDJDLQ OHIWPRUHURRPIRUVWXG\RQH\HWUDFNLQJDQGUHDGLQJFRGH )ROORZLQJLQWKHOLQHRIEHDFRQVLQ4X\LQ)DQUHOHDVHGWKHVWXG\�7KH(IIHFWV RI%HDFRQV&RPPHQWVDQG7DVNVRQ3URJUDP&RPSUHKHQVLRQ3URFHVVLQ6RIWZDUH    0DLQWHQDQFH� )DQ 7KLVVWXG\ZDVEXLOWXSRQ&URVE\HWDO�VSUHYLRXVUHVHDUFKLQ EHDFRQV7KHREVHUYDWLRQVRIWKHVWXG\VKRZHGWKDWFRPPHQWVZLWKLQSURJUDPPLQJFRGH KHOSHGWRLQFUHDVHFRPSUHKHQVLRQOHYHOVLIWKHFRGHLVIDPLOLDUWRWKHSURJUDPPHU:KHQ LWFDPHWREHDFRQV)DQQRWLFHGWKHVDPHSDWWHUQVDV&URVE\EHDFRQVW\SHVDUHLGHQWLILHG E\WKHLQGLYLGXDOSURJUDPPHU 7DNLQJDGLIIHUHQWGLUHFWLRQIURPWKHUHDGLQJDQGPDLQWHQDQFHDUHDRIFRGH FRPSUHKHQVLRQ D QHZ VWXG\ ZDV GRQH WR FRPSDUH EXJ WUDFNLQJ LQ WZR GLIIHUHQW SURJUDPPLQJODQJXDJHV6KDULIHWDOWRRNDORRNLQWRVRPHWKLQJWKDWKDGQ�WEHHQGRQH \HWDFRPSDULVRQRI&YHUVXV3\WKRQVRXUFHFRGHLQWKHFRQWH[WRIGHEXJJLQJ 7XUQHU )DOFRQH6KDULI /D]DU 7KHFKRLFHRIWKHWZRFRGLQJODQJXDJHVZHUHVLPSOH &LVDYHU\FRPPRQDQGFRQWLQXDOO\XVHGSURJUDPPLQJODQJXDJHDQGUHVHDUFKOHGWR VKRZLQJWKDW3\WKRQZDVVLPLODUWR&EXWHDVLHUDQGPRUHJUDFHIXO$QH\HWUDFNHUZDV XVHGWRFKHFNLIWKHUHZDVDGLIIHUHQFHLQH\HJD]HEHKDYLRUEHWZHHQQRYLFHVDQGQRQ QRYLFHVDFURVVERWKODQJXDJHVLIWKHUHZDVDGLIIHUHQFHLQYLVXDOHIIRUWDQGDOVRLIWKH SURJUDPPLQJODQJXDJHKDGDQLPSDFWRQHIILFLHQF\RIVROYLQJGLIIHUHQWWDVNVLQWKLVFDVH EXJILQGLQJDQGRYHUYLHZWDVNV  7KHVWXG\ZDVVHWXSXVLQJVWXGHQWVVWXG\LQJSURJUDPPLQJWKH\ZHUHEURNHQXS LQWRJURXSVGHSHQGHQWRQWKHLUDQVZHUVRIWKHLUNQRZOHGJHRIHDFKJLYHQSURJUDPPLQJ ODQJXDJH7KHWRWDOHQGHGXSKDYLQJPRUHSDUWLFLSDQWVLQWKH&SURJUDPPLQJJURXS WKLVLVFRUUHODWHGWRWKHDPRXQWRISURJUDPPLQJFRXUVHVWKDWWKHVWXGHQWVZHUHDEOHWRWDNH LQHDFKSURJUDPPLQJODQJXDJH7KHVWXG\LWVHOIFRQVLVWHGRIWKUHHGLIIHUHQWILQGEXJWDVNV DQGWZRRYHUYLHZWDVNV    7KHUHVXOWVWKDWZHUHDQDO\]HGZHUHDFFXUDF\WLPHIL[DWLRQFRXQWVDQGIL[DWLRQ GXUDWLRQIRUHDFKJURXS$FFXUDF\ZDVKLJKHUIRUDOOSDUWLHVLQWKH&JURXSDOVRLQWKH &JURXSQRYLFHVWRRNORQJHURYHUDOO)RUWKH&JURXSQRYLFHVKDGDKLJKHUIL[DWLRQ FRXQWZKHQFRPSDUHGWR3\WKRQ7KH3\WKRQJURXSWRRNORQJHUWRFRPSOHWHDOORIWKHLU WDVNV:LWKDOORIWKHGDWDDQDO\]HGWKHUHZDVQRVWDWLVWLFDOGLIIHUHQFHZKHQLWFDPHWR FRPSDULQJ&DQGS\WKRQZLWKDFFXUDF\DQGWLPHEXWWKHUHZDVDQRWLFHDEOHGLIIHUHQFH ZKHQLWFDPHWRIL[DWLRQUDWHRQEXJV%HLQJWKDWWKLVLVWKHILUVWVWXG\RILWVNLQGWKLVOHIW RSHQDIROORZXSVWXG\WKDWZLOOEHIRFXVLQJRQFRQVWUXFWVZLWKLQWKHWZRODQJXDJHVZKHUH WKHXVHUVJHWWRVHHERWKWKH&DQG3\WKRQFRGHV :LWKRXWKDYLQJFRQFUHWHHYLGHQFHRQH[SHGLWHGFRGHUHDGLQJIDFWRUVZHUHO\PRUH VWULFWO\RQWKHGDWDWKDWLVUHWULHYHGIURPH\HWUDFNLQJIRUSURJUDPFRPSUHKHQVLRQ7REH DEOHWRXQGHUVWDQGKRZFRPSUHKHQVLRQLVXQGHUVWRRGWKURXJKH\HWUDFNLQJGDWDZHPXVW ORRNDWKRZQDWXUDOODQJXDJHWH[WLVUHDGDQGXQGHUVWRRG5HDGLQJFDQEHDQDO\WLFDOO\ GLYLGHGLQWRSHUFHSWLRQDQGFRPSUHKHQVLRQ %XVMDKQ6FKXOWH %XVMDKQ  7RIXUWKHUORRNLQWRLGHQWLILHUVW\OHV6KDULIHWDOWRRNDQLQGHSWKORRNDWH\H PRYHPHQWVRQFDPHOFDVHYHUVXVXQGHUVFRUHV %LQNOH\HWDO6KDULI 0DOHWLF  7KHVWXG\ZDVDUHSOLFDWLRQRI%LQNOH\HWDO�VUHVHDUFKLQWKDWXVHGUHVSRQVH WLPHWRJHWGDWD6LQFHH\HWUDFNHUVDUHDEOHWRJDWKHUPRUHGDWDWKHKRSHZDVIRUDEHWWHU RYHUDOOORRNLQWRLGHQWLILHUVW\OHV  7KHVWXG\FRQVLVWHGRIHLJKWSKUDVHVIROORZHGE\DQLGHQWLILHUWKDWPDWFKHGZKDW WKH\VDZH[DFWO\7KHRWKHUWKUHHLGHQWLILHUVZHUHGLVWUDFWRUVWKDWFKDQJHWKHLGHQWLILHU 7KHJRDORIGRLQJH\HWUDFNLQJLQWKHVWXG\ZDVWRIRFXVDQGVHHLIWKHUHZDVDWULJJHUIRU    FRPSUHKHQVLRQZLWKLQDJLYHQWDVN8VLQJ$2,VWKHVWXG\DQGIRFXVZDVHDVLO\DEOHWREH QDUURZHGGRZQDQGDQDO\]HG  :LWKWKHXVHRIDQH\HWUDFNHULWZDVDEOHWRWHOOWKDWLGHQWLILHUVW\OHVLJQLILFDQWO\ DIIHFWVWLPHDVZHOODVWKHYLVXDOHIIRUWWKDWLWWDNHVWRFRUUHFWO\GHWHFWLGHQWLILHUVZLWKLQD SKUDVH:KLOH%LQNOH\�VVWXG\GLGQRWVKRZDUHODWLRQEHWZHHQWKHOHQJWKRIDSKUDVHDQG LWVVW\OHH\HWUDFNLQJGDWDUHSRUWHGWKDWFDPHOFDVHGWDVNVWRRNORQJHUWKDQXQGHUVFRUHG WDVNV$KLJKHUDFFXUDF\IRUFDPHOFDVHZDVIRXQGLQ%LQNOH\�VVWXG\&DPHOFDVLQJGLG QRWLFHDEO\DIIHFWWKHVSHHGRIQRYLFHVRYHUH[SHUWVWKRXJKLQWKLVVWXG\ZKLOHLQ%LQNOH\�V WKHUHZDVDGLUHFWFRUUHODWLRQWRWKHSURJUDPPHUV�IDPLOLDULW\ZLWKWKHLULGHQWLILHUVW\OH 7KHILQDOFRQFOXVLRQVEHLQJWKDWWKHUHPD\QRWKDYHEHHQDGLIIHUHQFHLQDFFXUDF\EXWWKHUH LVDODUJHLPSURYHPHQWZKHQLWFRPHVWRWLPHDQGYLVXDOHIIRUWZLWKXQGHUVFRULQJ 6KDULI 0DOHWLF  6LQFHFRGHPDLQWHQDQFHLVEHFRPLQJDODUJHUSDUWRISURJUDPPLQJHYHU\GD\LW�V LPSRUWDQWWRQRWMXVWIRFXVRQDSURJUDPPHU�VSRLQWRIYLHZEXWDOVRDFRGHUHYLHZHU�V SRLQWRIYLHZ2QHJURXSRISURJUDPPHUVZDQWHGWRWDNHDORRNGLUHFWO\DWWKHUHDGLQJRI FRGH8ZDQRHWDOWRRNDORRNLQRQWKHH\HPRYHPHQWVRIFRGHUHDGHUVE\GRLQJD FRGHUHYLHZVWXG\ 8ZDQR1DNDPXUD0RQGHQ 0DWVXPRWR 6RXUFHFRGH UHYLHZQRWMXVWE\WKHZULWHURIWKHFRGHEXWE\SHHUVKDVEHHQDSURYHQDQGORQJWLPH XVHGWHFKQLTXHWRLPSURYHFRGH7KHUHDUHVHYHUDOGLIIHUHQWW\ SHVRIFRGHUHYLHZLQFOXGLQJ $G+RF5HYLHZ3HUVSHFWLYH%DVHG5HDGLQJ&KHFNOLVW%DVHG5HDGLQJ'HIHFW%DVHG 5HDGLQJDQG8VDJH%DVHG5HDGLQJ:KLOHVRPHRIWKHGLIIHUHQWUHYLHZW\SHVKDYHEHHQ VKRZQWRSHUIRUPEHWWHURYHURWKHUVLWZDVQHYHUSURYHQYLDDFWXDOH\HPRYHPHQWV     7KHUHVHDUFKHUVVHWXSDZHOORUJDQL]HGVWXG\WKDWORRNHGDWOLQHUHDGLQJJD]HVDQG JODQFHVDQGOLQHLGHQWLILFDWLRQDQGGXUDWLRQ7KHUHVHDUFKHUVGHYHORSHGWKHLURZQVRIWZDUH WKDWZRUNHGZLWKWKHLUKDUGZDUHWKDWGLGDORWRIZRUNIRUWKHP,WZDVDEOHWRJHWWKHGDWD IURPWKHH\HWUDFNHUDQGILWLWLQWRWKHLUUHTXLUHPHQWV)LYHJUDGXDWHVSDUWLFLSDWHGLQWKH VWXG\(DFKSDUWLFLSDQWKDGVL[VPDOOSURJUDPVWKDWWKH\ZHUHWRUHYLHZ7RHQVXUHWKH YDOLGLW\RIWKHGDWDDPDQXDOFKHFNZDVGRQHRQGDWDREWDLQHGDQGLIPRUHWKDQSHUFHQW RIGDWDZDVEDGWKHZKROHWULDOZDVGLVFDUGHG  7KHVWXG\VKRZHGWKDWDFRPPRQSUDFWLFHLQFRGHUHDGLQJZDVWRUHYLHZWKHZKROH EORFNRIFRGHLQRQHFRQWLQXRXVUHDGLQJDQGWKHQORRNDWLQGLYLGXDOSDUWVRIFRGH7KH VWXG\DOVRVKRZHGDFRUUHODWLRQEHWZHHQFRGHUHDGLQJWLPHFRGHUHDGLQJSDWWHUQVDQGKRZ ZHOODFRGHUHYLHZHUZDVDEOHWRILQGDQHUURUZLWKLQWKHFRGH7KHUHVXOWVDOVRSURYHGWKDW FRGHUHYLHZHUVKDYHGLIIHUHQWSDWWHUQW\SHVZKHQUHDGLQJVXFKDVUHFXUVLYHO\JRLQJEDFN DQGORRNLQJDWUHODWHGYDULDEOHV:KLOHWKHVWXG\GLGQ�WVKRZDUHODWLRQVKLSEHWZHHQWKHVH UHDGLQJSDWWHUQVDQGSHUIRUPDQFHLWLVVWLOOLPSRUWDQWWRQRWHWKDWFRGHUHYLHZHUVGRUHDG LQDSDWWHUQZKLFKOHDYHVURRPIRUDODUJHUDQGPRUHLQGHSWKVWXG\RIFRGHUHYLHZHUVH\H PRYHPHQWV  )ROORZLQJRQWKHVDPHSDWKRI8ZDQR)DOFRQHDQG6KDULI 6KDULI)DOFRQH  0DOHWLF FRQGXFWHGDVWXG\ PLPLFNLQJWKDWRI8ZDQRHWDOEXWXVLQJPRUH SDUWLFLSDQWVZLWKYDULHGSURJUDPPLQJH[SHULHQFH7KHJRDORIWKHUHVHDUFKZDVWRRQFH DJDLQORRNDWKRZDSURJUDPPHUUHDGVFRGHZKHQJLYHQDVSHFLILFWDVNZLWKWKHHQGJRDO EHLQJWRUHFRJQL]HORJLFDOGHIHFWV     7KHVWXG\ORRNHGDWWZRGLIIHUHQWTXHVWLRQV)LUVWGLGWKHWLPHWDNHQWRVFDQDIIHFW DFFXUDF\WLPHDQGYLVXDOHIIRUWRIGHIHFWGHWHFWLRQ"6HFRQGGRHVWKHH[SHULHQFHRID SURJUDPPHUFRPHLQWRSOD\ZKHQGRLQJDFRGHUHYLHZ"7KHH[SHULPHQWZDVVHWXSXVLQJ DQRQLQWUXVLYHH\HWUDFNHU$OOYROXQWHHUVZHUHWDVNHGZLWKILQGLQJHUURUVLQIRXU GLIIHUHQWSLHFHVRIVRXUFHFRGHWKHVHFRGHVZHUHXVHGLQWKH8ZDQRVWXG\$VSHDNDORXG PHWKRGZDVXVHGDQGWKHSURJUDPPHUVRQFHJLYHQWKHVWLPXOLZHUHDVNHGWRWKLQNRXWORXG ZKHUHWKH\WKRXJKWWKHEXJVZHUHLQWKHFRGH2QFHDQDQVZHUZDVJLYHQWKHSURJUDPPHU PRYHGRQWRWKHQH[WVWLPXOXVDQGWKHWHVWFRQWLQXHG  7KHVWXG\LWVHOIYDULHGRQO\VOLJKWO\IURPWKH8ZDQRVWXG\7KHUHZHUH SURJUDPPHUVZLWKYDULHGH[SHULHQFHLQVWHDGRIRQO\ILYHZLWKWKHVDPHILHOGRIH[SHULHQFH 7KH8ZDQRVWXG\XVHGVL[SURJUDPVWRWDOZKLOHWKLVVWXG\XVHGIRXU7KH8ZDQRVWXG\ DOVRZDLWHGXQWLODSURJUDPPHUIRXQGWKHEXJRUIRUILYHPLQXWHVPD[LPXPWKLVVWXG\GLG QRW$OVRDGLIIHUHQWH\HWUDFNHUZDVXVHG  $OWKRXJKWKHUHZHUHPRGLILFDWLRQVIURPWKH8ZDQRVWXG\WKHVWXG\JLYHVJRRG UHVXOWV7KHVWXG\YDOLGDWHVWKDWVFDQWLPHGRHVSOD\DQLPSRUWDQWUROHLQWKHDPRXQWRI WLPHWKDWLWWDNHVWRGHWHFWHUURUVLQFRGH$OVRWKHYDULHGUDQJHRISURJUDPPHUVJDYHDQ LQVLJKWWRKRZH[SHUWSURJUDPPHUVDQGQRYLFHUHDGHUVWDNHDORRNDWFRGH([SHUWVWHQGWR KDYHORQJHUIL[DWLRQVZKLOHQRYLFHVWHQGWRWDNHEURDGHUVZHHSVRIWKHFRGHZKHQORRNLQJ IRUHUURUV1RWLFLQJWKDWWKHUHDUHGLIIHUHQWH\HPRYHPHQWSDWWHUQVOHDYHVURRPWRORRN GHHSHULQWRVFDQSDWWHUQVRQFRGHUHDGLQJDQGUHYLHZLQJ :KLOH FRGH UHDGLQJ LV DQ LPSRUWDQW SDUW RI FRGH FRPSUHKHQVLRQ FRGH VXPPDUL]DWLRQLVLPSRUWDQWDVZHOO+DYLQJDJRRGXQGHUVWDQGLQJRIZKDWDSURJUDPGRHV    KHOSVZLWKSURJUDPPDLQWHQDQFH6RPHWRROVFDQGRWKLVIRU\RXEXWWKH\PD\QRWDOZD\V EHEHVW7KHEDFNERQHRIDQ\DXWRPDWLFFRGHVXPPDUL]DWLRQLVLQWKHNH\ZRUGVDQG LGHQWLILHUVSURJUDPPHGLQWRLW,QDVWXG\ZDVFRPSOHWHGWKDWORRNHGWRXVHH\H PRYHPHQWVWREHWWHUFRGHVXPPDUL]DWLRQVRIWZDUH 5RGHJKHUR0F0LOODQ0F%XUQH\ %RVFK '�0HOOR  7KHVWXG\ZDVFRQGXFWHGE\KDYLQJSURJUDPPHUVUHDG-DYDPHWKRGVDQGWKHQ ZULWHWKHLURZQVXPPDULHVIRUWKHFRGH7KHUHVHDUFKZDVORRNLQJWRVHHKRZFORVH SURJUDPPHU�VFKRLFHRINH\ZRUGVDUHWRDYHFWRUVSDFHPRGHOWHUPIUHTXHQF\LQYHUVH GRFXPHQWIUHTXHQF\PRGHO 960WILGI $OVRGRSURJUDPPHUVIRFXVRQVLJQDWXUH FRQWUROIORZRULQYRFDWLRQVRIDPHWKRGDQ\PRUHWKDQRWKHUSDUWVRIWKHFRGH" 8VLQJDKLJKIUHTXHQF\H\HWUDFNHUWKHSURJUDPPHUVZHUHDVNHGWRUHDG-DYD PHWKRGVDQGZULWHVXPPDULHVIRUHDFKPHWKRG7KHFKRLFHVWKDWWKHSURJUDPPHUVPDGH ZHUHFRPSDUHGWRD960WILGIDSSURDFK7KHH\HWUDFNLQJUHVXO WVVKRZHGWKDWKDOIRIWKH WRSNH\ZRUGVIURP960ZHUHWKHVDPHRQHVWKDWZHUHPRVWFRPPRQO\UHDGE\WKH SURJUDPPHUV7KHH\HWUDFNLQJUHVXOWVDOVRVKRZHGWKDWSURJUDPPHUVVHHPHGWRSLFN VLJQDWXUHVRYHULQYRFDWLRQNH\ZRUGVLQ-DYDPHWKRGV:KLOHWKHSDSHUJRHVLQWRIXUWKHU GHWDLODERXWFRGHVXPPDUL]DWLRQDQGWKHWRROVFUHDWHGLWZDVIURPH\HWUDFNLQJGDWDWKDW WKH\ZHUHDEOHWRFRPHWRWKHLUFRQFOXVLRQV &RGHUHDGLQJDQGUHYLHZLQJKDVEHHQSURYHQWREHDPXOWLOHYHOHGFRPSUHKHQVLYH WDVN&RGHFRPSUHKHQVLRQLVEHFRPLQJDNH\SDUWRIXQGHUVWDQGLQJKRZDSURJUDPZRUNV $SURJUDPPHUVKRXOGEHDEOHWRXQGHUVWDQGZKDWSUREOHPDSURJUDPLVWU\LQJWRVROYHEH    DEOHWRUHDGWKHFRGHDQGDQDO\]HKRZLW�VVROYLQJWKHSUREOHPDVZHOODVNQRZKRZWR DGDSWFRGHWRWKHSUREOHPLWLVVROYLQJ 'LIIHUHQWPHWKRGVIRUDQDO\VLVRIH\HWUDFNLQJGDWDDUHEHLQJXVHGDQGZKLOHVRPH DUHEHFRPLQJQHDUO\XQLYHUVDOWKHUHLVDOZD\VDSRVVLELOLW\IRULPSURYHPHQWLQDQDO\VLV PHWKRGV7KHUHVXOWVRIRQHVXFKPHWKRG6FDQ0DWFKZHUHUHSRUWHGE\&ULVWLQRHWDOLQ  &ULVWLQR0DWK{W7KHHXZHV *LOFKULVW 7KHLUH[SHULPHQWZDVGHVLJQHGWR TXDQWLWDWLYHO\H[SODLQKRZVLPLODUWZRH\HPRYHPHQWVHTXHQFHVDUHE\XVLQJDOJRULWKPV ZLWKLQELRLQIRUPDWLFV  7KHLUH[SHULPHQWVFRQVLVWHGRIGDWDWKDWVKRZVWKHLUDSSURDFKLQUHODWLRQWRWKH /HYHQVKWHLQPHDVXUHZKLFKPHDVXUHVWKHGLVWDQFHEHWZHHQWZRVHTXHQFHV5HVXOWV VKRZHGWKDW6FDQ0DWFKRXWSHUIRUPHG/HYHQVKWHLQZKHQLWFDPHWRQRLV\GDWD7KHVHFRQG H[SHULPHQWZDVWRFKHFNWKHSHUIRUPDQFHRI6FDQ0DWFK*LYHQVWLPXOLDQGLQVWUXFWLRQV WRIROORZDQXPEHUSDWWHUQ6FDQ0DWFKSURYHGWRFOXVWHUDOORIWKHWULDOVVXFFHVVIXOO\DIWHU FRPSDULQJHYHU\RWKHUVHTXHQFHZLWK6FDQ0DWFK,QWKHWKLUGH[SHULPHQWKXPDQH\H PRYHPHQWVZHUHWHVWHGRQDVHDUFKWDVNWRWHVWLI6FDQ0DWFKZRXOGEHDEOHWRWHOOWKH GLIIHUHQFHLQFRQGLWLRQV2QHFRQGLWLRQKDGGLVWUDFWRUVLQZKLFKGLIIHUHQWFRORUVZHUH XVHG$IWHUFRPSOHWLQJWULDOVHDFKSDUWLFLSDQWZDVDQDO\]HGRQDQLQGLYLGXDOEDVLV 6FDQ0DWFKSURYHGWRKDYHDKLJKHUQRUPDOL]HGVFRUHDFURVVWDVNV  7KH\FRQFOXGHGWKDW6FDQ0DWFKZDVDEOHWRRXWSHUIRUPSUHYLRXVVWULQJHGLW PHWKRGV6FDQ0DWFKLVDEOHWRWHOOWKHGLIIHUHQFHLQFRQGLWLRQVDQGDEOHWRZLWKVWDQGDQG ZRUNZLWKQRLV\GDWD:LWKWKHXVHRI6FDQ0DWFKWKHGDWDUHFHLYHGIURPH\HWUDFNHUVDUH DEOHWREHEHWWHUDQDO\]HGZLWKPRUHSUHFLVLRQ    2QHPDMRUHQKDQFHPHQWWRH\HWUDFNLQJDQDO\VLVZDVWKHLQWURGXFWLRQRI$UHDVRI ,QWHUHVWRU$2,V$Q$2,LVFRPPRQO\DUHFWDQJOHWKDWFRYHUVDOLQHRIFRGH(DFK LQGLYLGXDOOLQHJHWVDQ$2,DQGHDFK$2,KDVLWVRZQLGHQWLILHU%\XVLQJDQ$2,LWLV HDV\WRPRQLWRUWKHH\HPRYHPHQWVZLWKLQDVSHFLILFOLQHRUVHJPHQWRIFRGHDOORZLQJD QDUURZLQJGRZQWRDVSHFLILFNH\ZRUGRUVHFWLRQRIFRGH,IWKHUHLVDWOHDVWRQHIL[DWLRQ ZLWKLQWKH$2,WKH$2,LVFRQVLGHUHGKLWRWKHUZLVHWKH$2,LVPLVVHG7KHXVHRIDQ$2, QRWRQO\KHOSVWREHWWHURUJDQL]HGDWDEXWGUDVWLFDOO\FDXVHVDUHGXFWLRQLQVFDQSDWKGDWD $VFDQSDWKLVWKHFKURQRORJLFDORUGHURI$2,VRUIL[DWLRQV  (\HWUDFNLQJ6WXGLHVLQ&RPSXWHU(GXFDWLRQ 7HUHVD%XVMDKQDQG&DUVWHQ6FKXOWHGHVFULEHWKHVW\OHRIUHDGLQJFRGHDVDWHDFKLQJ WRROLQVWHDGRIOLPLWLQJWRRQO\ZULWLQJDVDZD\RIOHDUQLQJWRSURJUDP %XVMDKQ 6FKXOWH E 7KH\VWXGLHGWKHSRWHQWLDOWRIDFLOLWDWHFRGHUHDGLQJWKHDVSHFWVRIFRGHUHDGLQJ WKDWDUHZRUWK\RIEHLQJUHDGLQTXLULHVDVWRZKDWVKRXOGEHNQRZQDERXWFRGHUHDGLQJ DQGZKDWFRPHVWRPLQGZKHQWKLQNLQJDERXWFRGHUHDGLQJ$IWHUWDVNLQJSDUWLFLSDQWV VHYHUDOGLIIHUHQWDVSHFWVRIFRGHUHDGLQJZHUHEURXJKWWROLJKW,WZDVOHDUQHGWKDWFRGH UHDGLQJKDVDGLUHFWOLQNWRFRPSUHKHQVLRQRIDOODVSHFWVRISURJUDPPLQJLQFOXGLQJ DOJRULWKPVDQGVHPDQWLFVRIFRQVWUXFWV7KLVOHDYHVURRPRSHQWRIXUWKHUVWXG\FRGH UHDGLQJDQGWHFKQLTXHV 7R FRQILUP WKDW H\H WUDFNLQJ ZDV D SODXVLEOH UHVRXUFHWR XVH LQFRPSXWLQJ HGXFDWLRQDFDVHVWXG\DQGZRUNVKRSZHUHFRQGXFWHGWKDWZRXOGKHOSGHWHUPLQHWKHXVDJH RIH\HWUDFNHUVDQGWKHTXDOLW\RIGDWDWKDWFRXOGEHREWDLQHG,WZDVGHWHUPLQHGWKDWH\H WUDFNLQJGLGDOORZIRUDGHHSHUORRNLQWRKRZDQRYLFHUHDGVDQGXQGHUVWDQGFRGH,WZDV    DOVRGHFLGHGWKDWWKHGHWDLORIGDWDWKDWLVIRXQGIRUPH\HWUDFNLQJUHVHDUFKFDQKHOSWR DGYDQFHGLIIHUHQWOHDUQLQJWRROVIRUQHZSURJUDPPHUV7KHLQIRUPDWLRQJDWKHUHGLVQ�W VWULFWO\UHODWHGWRQRYLFHVWKRXJK)URPWKHZRUNVKRSH[DPSOHVWKDWZHUHJDWKHUHGWKHUH ZDVDQRWLFHLQGLIIHUHQFHLQWKHZD\WKDWH[SHUWSURJUDPPHUVUHDGFRGHDVZHOO.QRZLQJ WKLVLQIRUPDWLRQFRXOGJLYHIXUWKHUGLUHFWLRQWRRQH�VRZQDSSURDFKZKHQLWFRPHVWR UHDGLQJDQGZULWLQJSURJUDPPLQJ7KHSDSHUFRQFOXGHVWKDWH\HWUDFNLQJLVDJUHDWILHOG WRUHVHDUFKIRUWHDFKLQJFRPSXWLQJ(\HWUDFNHUVDUHDEOHWRJLYHGHWDLOHGGDWDLQWRKRZ LQGLYLGXDOSURJUDPPHUVZRUNDVZHOODVKHOSWRVWXG\WKHGLIIHUHQWPHWKRGVRIFRPSXWLQJ HGXFDWLRQ %XVMDKQHWDO 0RUHLQIRUPDWLRQDERXWWKLVVWXG\LVJLYHQEHORZ  2YHUWKHWLPHRIRQHSURJUDPPLQJFRXUVHLWLVSRVVLEOHIRUDQRYLFH�VH\H PRYHPHQWVWRFKDQJH7KHUHFDQEHVHYHUDOGLIIHUHQWSDWWHUQVRIFRGHUHDGLQJQDPHO\ -XPS&RQWURO/LQHDU/LQH6FDQ6FDQ)OLFNLQJ7KUDVKLQJDQG6LJQDWXUHV7KHVHFRYHU DOOFDWHJRULHVRIJD]HSDWWHUQVIURPQDWXUDOWH[WUHDGLQJ OHIWWRULJKWWRSWRERWWRP WR JD]HVIRFXVLQJRQPHWKRGVRUJD]HVMXPSLQJDOORYHUWKHFRGHZLWKQRUHFRJQL]DEOH SDWWHUQ7KH\DOVRGHVFULEHGLIIHUHQWVWUDWHJLHVRIUHDGLQJ'HVLJQ$W2QFH'HEXJJLQJ 3URJUDP)ORZ7HVW+\SRWKHVLV7UDLO (UURUDQG)ORZ&\FOH7KHVHVWUDWHJLHVKHOSDQ DQDO\VWGHWHUPLQHZKDW�VKDSSHQLQJLQWKHSURJUDPPHU�VPLQGVXFKDV3URJUDP)ORZ ZKLFKGHVFULEHVWKHIROORZLQJRIWKHH[HFXWLRQRUGHURIWKHFRGHRU'HEXJJLQJZKLFK QRWLFHVH\HJD]HVORRNDWDOOHOHPHQWVHYHQO\VXJJHVWLQJWKDWWKHUHDGHULVWU\LQJWRILQG HUURUV7KHUHVHDUFKWDNHVDORRNDWRQHSURJUDPPHURYHUWKHOHQJWKRIDSURJUDPPLQJ FRXUVHDQGQRWLFHVWKDWH\HPRYHPHQWVFKDQJHTXLWHDELW     7KHILUVWSDUWRIWKHUHVHDUFKZDVDWWKHEHJLQQLQJRIWKHFRXUVH7KHVWXGHQWUHDG WKURXJKWKHLUFRGHLQDOLQHDUPDQQHUOHIWWRULJKWWRSWRERWWRP7KHVHFRQGUHFRUGLQJ ZDVGXULQJWKHPLGGOHRIWKHFRXUVHDQGDOUHDG\FKDQJHVLQFRGHUHDGLQJSDWWHUQVZHUH YLVLEOH:KLOHWKHSDUWLFLSDQWVWDUWHGLQDOLQHDUIDVKLRQWKHUHZHUHPRUHUHJUHVVLRQVDQG WKHVHUHJUHVVLRQVVHHPHGWREHIRFXVHGRQYDULDEOHV7KLVVXJJHVWVWKDWWKHUHDGHUZDV FKHFNLQJWRIROORZWKHKLVWRU\RIWKHYDULDEOH$WWKHHQGRIWKHFRXUVHDWKLUGUHFRUGLQJ ZDVFRPSOHWHG$JDLQWKHSDUWLFLSDQWVWDUWHGRIILQDOLQHDUIDVKLRQEXWVRRQMXPSHG VWUDLJKWWRWKHPHWKRGRIWKHSURJUDPDIWHUJDWKHULQJHQRXJKLQIRUPDWLRQWRXQGHUVWDQG WKHFRGHWKHUHDGHU�VH\HPRYHPHQWVWKHQVWDUWWRPRYHLQDUDQGRPSDWWHUQ  $OWKRXJKWKHWHVWSDUWLFLSDQWZDVRQO\RQHLQGLYLGXDOWKHGDWDRQWKHJD]HDQGH\H PRYHPHQWVZHUHHQRXJKWRSURYHWKDWZLWKLQDVKRUWSHULRGRIWLPHDUHDGHU�VH\H PRYHPHQWVFDQFKDQJHTXLWHDELW5HJUHVVLRQVWKDWZHUHVHHQZHUHSDUWRIDFRQILUPDWLRQ VWUDWHJ\WKHWHVWSDUWLFLSDQWZDVPDNLQJVXUHWKDWWKHLUSURJUDPZDVGRLQJWKDWWKH\ WKRXJKW7KLVFRXOGEHDQLVVXHGXHWRWKHIDFWWKDWSURJUDPVGRQRWDOZD\VZRUNLQD QDWXUDOWH[WIDVKLRQVRFRQILUPLQJLQDOLQHDURUGHULVQ�WDOZD\VJRLQJWRZRUN7KHGDWD GRHVFRPHWROLJKWWKRXJKDQGEULQJLQWRFRQVLGHUDWLRQWKDWDQ\QRYLFHSURJUDPPHUWKDW FKDQJHVIURPVWRU\UHDGLQJWRFRGHUHDGLQJFRXOGEHDSRVVLEOHVWHSRQWKHZD\WREHFRPLQJ DEHWWHUFRGHUHDGHU :LWKDQHYHUJURZLQJLQWHUHVWLQWHDFKLQJFRPSXWHUVFLHQFHDQGILQGLQJZD\VWR XQGHUVWDQGFRPSXWHUVFLHQFHSURJUDPFRPSUHKHQVLRQLVEHFRPLQJPRUHDQGPRUHDPDLQ SOD\HULQWKHILHOGRIHGXFDWLRQ,Q6FKXOWHHWDOUHYLHZHGSURJUDPFRPSUHKHQVLRQ IRU&6HGXFDWRUV 6FKXOWH&OHDU7DKHUNKDQL%XVMDKQ 3DWHUVRQ 7KHUHYLHZLV    H[SODLQHGIURPD&6HGXFDWRU�VSHUVSHFWLYH3URJUDPFRPSUHKHQVLRQLVEURNHQGRZQLQWR VHFWLRQVH[WHUQDOUHSUHVHQWDWLRQFRJQLWLYHVWUXFWXUHDQGWKHDVVLPLODWLRQSURFHVV$Q\ SDUWRIDSURJUDPWKDWLVQRWDOUHDG\NQRZQE\WKHSURJUDPPHULVFRQVLGHUHGDQH[WHUQDO UHSUHVHQWDWLRQZKLOHFRJQLWLYHVWUXFWXUHSHUWDLQVWRWKHDOUHDG\NQRZQLQIRUPDWLRQWKDW HDFKSURJUDPPHUSRVVHVVHV7KHILQDOVWHSDVVLPLODWLRQLVWKHSURFHVVRIJRLQJDERXW WU\LQJWRVWUDWHJL]HDQGVROYHWKHSURJUDPPLQJSURFHVVDWKDQG  7KHUHVHDUFKJRHVLQWRGLIIHUHQWPRGHOVRISURJUDPFRPSUHKHQVLRQWKDWUHSUHVHQW GLIIHUHQWIRFXVHVRIWKRXJKWZLWKLQSURJUDPFRPSUHKHQVLRQ7KHGLIIHUHQWPRGHOVLQFOXGH WKH%ORFN0RGHO6RORZD\DQG(KUOLFK3HQQLQJWRQ/HWRYVN\DQGPRUH7KHUHVHDUFK JRHVLQWRHPSLULFDOZRUNFKDUDFWHUL]DWLRQH[WHUQDOUHSUHVHQWDWLRQVDVVLPLODWLRQFRJQLWLYH VWUXFWXUHDQGPDSSLQJWR%ORFN0RGHO7KHUHVHDUFKVKRZVFRPPRQSDUWIRUHDFK GLIIHUHQWPRGHO)RUH[DPSOHUHSUHVHQWDWLRQRIFRPSUHKHQVLRQDQGDVVLPLODWLRQZHUH VLPLODUIRUVHYHUDOGLIIHUHQWPRGHOV  6FKXOWHFRQFOXGHVZLWKWKHSULPDU\IRFLRIWKHGLIIHUHQWPRGHOV7KHILUVWEHLQJ GRPDLQNQRZOHGJHIRUFRPSUHKHQVLRQEHLQJDQXQGHUHVWLPDWHGZD\RIWHDFKLQJ7KH VHFRQGEHLQJWKDWWKHUHDUHVHYHUDOGLIIHUHQWSRVVLEOHOHDUQLQJWDVNVIRUUHDGLQJDQG FRPSUHKHQGLQJSURJUDPV3OXVWKHSRVVLEOHXVHRILQVWUXPHQWVFRXOGEHKHOSIXOIRU OHDUQLQJ)LQDOO\H[SHUWVKDYHDIOH[LEOHDQGQDYLJDWLRQDOPHQWDOUHSUHVHQWDWLRQRIZKDWD SURJUDPGRHVE\UHDGLQJFRGHDQGNQRZOHGJHOHDUQHGE\QRYLFHSURJUDPPHUVFDQEH OLNHQHGWRDSDWFKHVEHLQJVHZQWRJHWKHU     2WKHU%LRPHWULFEDVHG6WXGLHVRQ3URJUDP&RPSUHKHQVLRQ 6LQFHWKHFRPSUHKHQVLRQRIQDWXUDOWH[WKDVEHHQXQGHUVWRRGIRUGHFDGHV*HWWLQJ LQWR WKH PLQG RI D SURJUDPPHU LV WKH EHVW ZD\ WR XQGHUVWDQG KRZ SURJUDPV DUH FRPSUHKHQGHG6LHJPXQGHWDOWRRNSURJUDPFRPSUHKHQVLRQWRDQHZOHYHOZKHQWKH\ UDQH[SHULPHQWVXVLQJIXQFWLRQDOPDJQHWLFUHVRQDQFHLPDJLQJ I05, RQSURJUDPPHUV ZKLOHUHDGLQJGLIIHUHQWFRGHVHJPHQWV7KHJRDORIWKHVWXG\ZDVWRILQGWKHGLIIHUHQWSDUWV RIWKHEUDLQWKDWDUHDFWLYDWHGZKHQDSURJUDPPHULVUHDGLQJDQGSURFHVVLQJFRGH  3UHSDUDWLRQIRUWKHVWXG\ZDVLPSRUWDQWZKHQLWFDPHWRGRLQJWKHI05,VWXG\ $YRLGLQJELDVHVZDVNH\DQGJLYHQOLWWOHZLJJOHURRPWKHVRXUFHFRGHVQLSSHWVKDGWR UHPDLQVPDOOZKLFKZDVDGLIILFXOWWDVNEHFDXVHWKHSURJUDPVKDGWRKDYHHQRXJKRID FKDOOHQJHWRDOORZIRUWKHI05,WRUHDGEUDLQDFWLYLW\7KLVOLPLWHGWKHSURFHVVWRRQO\ FRPSUHKHQGLQJRQHDVSHFWRIFRPSOH[FRPSUHKHQVLRQEXWVWLOOOHIWDQRSHQLQJIRUVWXGLHV WRIROORZ  7KH VWXG\ VKRZHG WKDW GLIIHUHQW DUHDV RI WKH EUDLQ WKDW SURFHVVHG DWWHQWLRQ ODQJXDJHPHPRU\FDWHJRUL]DWLRQSUREOHPVROYLQJYHUEDODQGQXPHULFDOPHPRU\DQG UHDGLQJDOOFDPHLQWRSOD\ZKHQDSURJUDPPHUZDVUHDGLQJDQGVROYLQJFRGH5HDGLQJ VRXUFHFRGHVWDWHPHQWVVKRZHGFORVHWRUHDGLQJVHQWHQFHVLQWKDWWKHSDUWRIWKHEUDLQZDV DFWLYHWKDWLVXVHGIRUVLOHQWZRUGUHDGLQJ3UREOHPVROYLQJZDVDOVRWULJJHUHGGXHWRLW EHLQJ VLPLODU WR GLIIHUHQW SV\FKRORJLFDO WDVNV LQYROYLQJ DEVWUDFWLQJ SDWWHUQV DQG FRQVWUXFWLQJKRZWKH\ZRUN9HUEDODQGQXPHULFPHPRU\ZHUHRIFRXUVHLPSRUWDQW NHHSLQJPDQLSXODWHGYDULDEOHVLQPHPRU\ZRXOGDFWLYDWHWKLVSDUWRIWKHEUDLQ     7KHFRQFOXVLRQRIWKHI05,VWXG\EURXJKWWROLJKWWKHQHXURVFLHQFHDVSHFWRI SURJUDPFRPSUHKHQVLRQ:KLOHKDYLQJEHHQXVHGIRUGHFDGHVLWLVQRZFOHDUWKDWGLIIHUHQW SDUWVRIWKHEUDLQDUHDFWLYDWHGZKHQUHDGLQJDQGFRPSUHKHQGLQJFRGH7KHVHILYHSDUWVRI WKHEUDLQDOOILWWKHERWWRPXSSURJUDPFRPSUHKHQVLRQPHWKRG7KHVWXG\DOVRFRQILUPHG 'LMNVWUD�VQRWLFHRIODQJXDJHSURFHVVLQJEHLQJDQLQWHJUDOSDUWRISURJUDPFRPSUHKHQVLRQ :LWKRXWDQXQGHUVWDQGLQJRIODQJXDJHSURFHVVLQJWKHUHFDQEHDOOVRUWVRISUREOHPVZKHQ LWFRPHVWRFRPSUHKHQGLQJFRGH7KHVWXG\ZKLOHVPDOOSURYHGDFFXUDWHZLWKFOHDUUHVXOWV ZKLFKOHIWRSHQWKHGRRUIRUPRUHQHXURSV\FKRORJLFDOVWXGLHVRQFRGHFRPSUHKHQVLRQ 6LHJPXQGHWDO  %HLQJDEOHWRORRNLQWRKRZDSURJUDPPHUWKLQNVFDQDVVLVWLQWKHSUHYHQWLRQRI EXJVEHIRUHDSURJUDPLVHYHUHYHQFRPSLOHG%HJHOHWDOGLGUHVHDUFKLQWRXVLQJSV\FKR SK\VLRORJLFDOVHQVRUVWRGHWHUPLQHDSURJUDPPHU�VPHQWDOVWDWHZKHQSURJUDPPLQJ )ULW] %HJHO0�OOHU<LJLW(OOLRWW =�JHU 7KH\ORRNDORRNDWXVLQJH\HWUDFNLQJ('$ DQG((*VHQVRUVWRVHHLIWKH\FRXOGSUHGLFWLIDWDVNKDGGLIIHUHQWGLIILFXOWLHVZKLFK VHQVRUVZRUNHGEHVWDQGFDQWKH\XVHSV\FKRSK\VLRORJLFDOPHDVXUHWRSUHGLFWWDVN GLIILFXOW\ZKLOHWKHSURJUDPPHULVZRUNLQJ  7KHLUH[SHULPHQWFRQVLVWHGRISURIHVVLRQDOSURJUDPPHUVHDFKSURJUDPPLQJ HLJKWGLIIHUHQWFRPSUHKHQVLRQWDVNV'DWDZDVJDWKHUHGIURPWKHSDUWLFLSDQWVXVLQJ GLIIHUHQWVHQVRUVH\HWUDFNLQJ('$DQG((*9LGHRRIERWKWKHSDUWLFLSDQWDVZHOODVD VFUHHQFDSWXUHDQGDXGLRUHFRUGLQJVZHUHSHUIRUPHGDVZHOO$IWHUHDFKWDVNWKH SDUWLFLSDQWVZHUHJLYHQDVXUYH\WRILOORXWVHYHUDOGLIIHUHQWTXHVWLRQVPHQWDOSK\VLFDO DQGWHPSRUDOGHPDQGVSHUIRUPDQFHHIIRUWDQGIUXVWUDWLRQ     7KHUHVXOWVFDPHWRFRQFOXGHWKDWUHJXODUVWUHVVFDQFDXVHDSURJUDPPHUWRFUHDWH EXJVLQWKHLUVRIWZDUH7KHVHVWUHVVOHYHOVFDQEHSUHGLFWHGXVLQJSV\FKRSK\VLRORJLFDO VHQVRUVZLWKSUHFLVLRQRIRYHUDUHFDOO7KHWHVWDOVRZDVDEOHWRVKRZWKDWD PXOWLWXGHRIVHQVRUVGRHVQ�WKDYHWREHXVHGWRVKRZWKHVHUHVXOWV7KHGDWDIRXQGKDV JLYHQDQH[WUDLQVLJKWLQWRKHOSLQJGHYHORSWRROVWKDWFRXOGSRVVLEO\SUHYHQWDSURJUDPPHU IURPLQWURGXFLQJPRUHEXJVLQWRWKHLUFRGH  'LVFXVVLRQ :LWKWKHFRQWLQXDOFRVWRIPDLQWDLQLQJFRGHWKHUHDUHIXUWKHUVWXGLHVWKDWQHHGWREH GRQHWRKHOSDOOHYLDWHWKHZHLJKWRIFRGHPDLQWHQDQFH:LWKVWXGLHVVKRZLQJWKDW PDLQWHQDQFHLVDWOHDVWRIWKHWLPHVSHQWSURJUDPPLQJLVRQPDLQWHQDQFHDQGUHDGLQJ FRGHLWLVLPSRUWDQWWKDWVWXGLHVIRFXVRQNH\HOHPHQWVWKDWDUHOLQNHGWRUHGXFLQJWKH DPRXQWRIWLPHVSHQWUHDGLQJFRGH6LQFHUHDGLQJFRGHLVDFRPSUHKHQVLYHWDVNDQGH\H WUDFNLQJLVWKHSULPDU\OLQNWRUHDGLQJZHQHHGWRWDNHDORRNDWKRZQDWXUDOFRPSUHKHQVLRQ RIWH[WLVGRQH7RGRWKLVZHQHHGWRFRPSDUHKRZXVHUVUHDGDQGFRPSUHKHQGQDWXUDO WH[WYHUVXVKRZWKH\UHDGSURJUDPPLQJFRGH :LWKDQHZLQVLJKWRQWKHSURFHVVRIUHVHDUFKIURP.LWFKHQKDP HYLGHQFHEDVHG VRIWZDUHHQJLQHHULQJ  .LWFKHQKDPHWDO DQGHDVLHUDFFHVVWRH\HWUDFNHUVZHKDYH VHHQDFRQWLQXDOJURZWKLQH\HWUDFNLQJUHVHDUFKZKHWKHUWKHUHVHDUFKLVRQFRGHUHDGLQJ FRGHFRPSUHKHQVLRQSV\FKRSK\VLRORJLFDOHYDOXDWLRQRUH\HPRYHPHQWVLQJHQHUDO$V ORQJDVWKHUHLVDSRVVLELOLW\WRILQGWKHFRQQHFWLRQEHWZHHQH[SHUWFRGHUHDGHUVDQGQRYLFH FRGHUHDGHUVZHZLOOFRQWLQXHWRXVHH\HWUDFNLQJWRKHOSXVILQGWKHSULPDU\FRQQHFWLRQV    EHWZHHQWKHWZRDVZHOODVVWXG\QDPLQJFRQYHQWLRQVSURJUDPPLQJFRPSUHKHQVLRQDQG EXJILQGLQJSURFHVVHV 7KHSUHYLRXVVWXGLHVGLGQRWFRQVLGHUWRTXDQWLI\OLQHDULW\EHWZHHQVRXUFHFRGHDQG QDWXUDOODQJXDJHWH[WVEHVLGHVRXUYHU\UHFHQWZRUNUHODWHGWR-DYDFRGH %XVMDKQHWDO  :HDWWHPSWHGWRH[WHQGZKDWZDVGRQHLQ %XVMDKQHWDO ZLWK&     7+(678'< 7KLVFKDSWHUSUHVHQWVWKHGHWDLOVRIWKHVWXG\:HGHVFULEHRXUH[SHULPHQWDOGHVLJQ WDVNVDQGVWLPXOLSDUWLFLSDQWUHFUXLWLQJPHDVXUHVZHXVHGDVZHOODVDGHWDLOHGGHVFULSWLRQ RIWKHVWXG\LQVWUXPHQWDWLRQ7KHDSSHQGL[FRQWDLQVDFRPSOHWHOLVWLQJRIDOOVWXG\ PDWHULDOV  6WXG\2YHUYLHZ 7KLVVWXG\VHHNVWRDQDO\]HDQGFRPSDUHWKHUHDGLQJDQGFRPSUHKHQVLRQRIQDWXUDO ODQJXDJHWH[WDQG&VRXUFHFRGH7KHPDLQSXUSRVHVRIWKLVVWXG\DUHWRGHWHUPLQHLI QDWXUDOWH[WLVUHDGGLIIHUHQWO\IURPVRXUFHFRGHDVZHOODVLIQRYLFHSURJUDPPHUVUHDG GLIIHUHQWO\WKDQQRQQRYLFHV:HFRQVLGHUDQRQQRYLFHWREHDVWXGHQWZKRZDVH[SRVHG WRSURJUDPPLQJLQDSULRUFRXUVH7KH\DUHW\SLFDOO\VWXGHQWVHQUROOHGLQDKLJKHUOHYHO FRXUVH1RYLFHVRQWKHRWKHUKDQGKDGOLWWOHRUQRSURJUDPPLQJEDFNJURXQG  ,QGHSHQGHQWDQG'HSHQGHQW9DULDEOHV 7KHLQGHSHQGHQWYDULDEOHLQWKLVVWXG\LVWKHW\SHRIVWLPXOXVEHLQJSUHVHQWHG VRXUFHFRGH 6& RUQDWXUDOODQJXDJHWH[W 17 :HKDGVHYHQVRXUFHFRGHSURJUDPVDQG WKUHH(QJOLVKODQJXDJHWH[WVWREHWHVWHG7KHGHSHQGHQWYDULDEOHVZHFROOHFWDUHUHODWHGWR RXUUHVHDUFKTXHVWLRQV7DEOHGHVFULEHVWKHLQGHSHQGHQWDQGGHSHQGHQWYDULDEOHVIRUWKH VWXG\      7DEOH2YHUYLHZRIWKH,QGHSHQGHQWDQG'HSHQGHQW9DULDEOHV ,QGHSHQGHQW9DULDEOHV x 7\SHRIVWLPXOXV 6&RU17  x ([SHUWLVH 1RYLFHRU1RQQRYLFH  'HSHQGHQW9DULDEOHV x /LQHDULW\0HWULFV o 9HUWLFDO1H[W7H[W o 9HUWLFDO/DWHU7H[W o +RUL]RQWDO/DWHU7H[W o 5HJUHVVLRQ5DWH o /LQH5HJUHVVLRQ5DWH x 6DFFDGH/HQJWK x (OHPHQW&RYHUDJH  7KHGHSHQGHQWYDULDEOHVDUHH[SODLQHGEHORZ7KHVHPHDVXUHVDUHDOOEDVHGRQ ZKHUHWKHSHUVRQIL[DWHGDWDQGWKHWLPHWKH\IL[DWHGRQWKDWORFDWLRQ LQRWKHUZRUGVWKH IL[DWLRQFRXQWVDQGWKHIL[DWLRQGXUDWLRQV :HXVHH\H&RGH 0+DQVHQ WRPDS WKHJD]HFRRUGLQDWHVWROLQHDQGZRUGSRVLWLRQVLQWKHVWLPXOLIRUERWK17DQG6&7KHVH KDYHEHHQDGDSWHGIURP %XVMDKQHWDO  x 9HUWLFDO1H[W7H[W  7KHSHUFHQWDJHRIIRUZDUGVDFFDGHVWKDWHLWKHUVWD\RQWKH VDPHOLQHRUPRYHRQHOLQHGRZQ x 9HUWLFDO/DWHU7H[W  7KHSHUFHQWDJHRIIRUZDUGVDFFDGHVWKDWHLWKHUVWD\RQWKH VDPHOLQHRUPRYHGRZQDQ\QXPEHURIOLQHV x +RUL]RQWDO/DWHU7H[W  7KHSHUFHQWDJHRIIRUZDUGVDFFDGHVZLWKLQDOLQH    x 5HJUHVVLRQ5DWH  7KHSHUFHQWDJHRIEDFNZDUGVDFFDGHVRIDQ\OHQJWK x /LQH5HJUHVVLRQ5DWH  7KHSHUFHQWDJHRIEDFNZDUGVDFFDGHVZLWKLQDOLQH x 6DFFDGH/HQJWK$YHUDJH(XFOLGHDQGLVWDQFHEHWZHHQHYHU\VXFFHVVLYHSDLURI IL[DWLRQV x (OHPHQW&RYHUDJH  )UDFWLRQRIZRUGVWKHSDUWLFLSDQWORRNHGDW 7KHILUVWILYHPHDVXUHVLQGLFDWHUHDGLQJEHKDYLRUDQGZLOOEHPDLQO\XVHGWR FRPSDUHWKH6&WRWKH17�VZKLFKDUHUHODWHGWR547KH\DUHEDVHGRQIL[DWLRQVDQGWDNH LQWRDFFRXQWERWKKRUL]RQWDODQGYHUWLFDOUHDGLQJ 7KHODVWWZRPHDVXUHVLQGLFDWHGLIIHUHQFHVEHWZHHQQRQQRYLFHVDQGQRYLFHV1RQ QRYLFHVIRUH[DPSOHWHQGWRKDYHORQJHUVDFFDGHVWKDQQRYLFHVDQGDOVRWHQGWRIRFXVRQ UHOHYDQWSDUWVRIWKHFRGHLHHOHPHQWFRYHUDJHLVVPDOOHU  3DUWLFLSDQWV )RUWKLVVWXG\ZHUHFUXLWHGDWRWDORISDUWLFLSDQWVDOORIZKRPZHUHVWXGHQWVDW <RXQJVWRZQ6WDWH8QLYHUVLW\3DUWLFLSDQWVZHUHUHFUXLWHGRIWKHLURZQIUHHZLOOZLWKWKH RSWLRQWRRSWRXWRIWKHVWXG\DWDQ\WLPH7KHSDUWLFLSDQWVUDQJHGIURP\HDUVRI SURJUDPPLQJH[SHULHQFHWR\HDUVRUJUHDWHUSURJUDPPLQJH[SHULHQFH:HZDQWHGWR FDSWXUHDEURDGUDQJHRISURJUDPPHUVVRDVWRKHOSZLWKRXUVHFRQGUHVHDUFKTXHVWLRQ54 :HEURNHGRZQSDUWLFLSDQWVLQWRWZRVHSDUDWHJURXSVGHWHUPLQHGE\WKHLUH[SHULHQFHLQ SURJUDPPLQJ6WXGHQWVZKRKDGWDNHQKLJKHUOHYHOFODVVHVDQGRUKDYHSURJUDPPHG SURIHVVLRQDOO\ ZHUH FODVVLILHG DV QRQQRYLFHV  :KLOH WKRVH LQ WKH ,QWURGXFWLRQ WR    3URJUDPPLQJFODVVZHUHFODVVLILHGDVQRYLFHV1RWHWKDWPRVWRIWKHQRYLFHVGLGQRWKDYH DQ\NLQGRISURJUDPPLQJEDFNJURXQGSULRUWRWDNLQJWKH,QWURGXFWLRQWR3URJUDPPLQJ FODVV,QWRWDOWKHUHZHUHQRQQRYLFHVSURJUDPPHUVDQGQRYLFHSURJUDPPHUV 7KHVWXGHQWVSDUWLFLSDWLQJIURPWKH,QWURGXFWLRQWR3URJUDPPLQJFODVVZHUHJLYHQ WKLVVWXG\DVDQH[WUDFUHGLWDVVLJQPHQW7KH\ZHUHWHVWHGGXULQJZHHNDQGDIWHUWKH VHPHVWHUVWDUWHG7KLVPHDQVWKDWWKH\KDGVL[ZHHNVWROHDUQ&QRWDWLRQDVZHOODVWKH FRQFHSWVVXFKDVGDWDW\SHVDULWKPHWLFRSHUDWRUVYDULDEOHVGHFODUDWLRQVDVVLJQPHQWDQG LQLWLDOL]DWLRQVHOHFWLRQVWUXFWXUHVDQGORRSLQJVWUXFWXUHV 7KHSDUWLFLSDQWVILOOHGRXWDSUHTXHVWLRQQDLUHEHIRUHWKH\GLGWKHVWXG\7KHUHZHUH IHPDOHDQGPDOHVWXGHQWV7KHDJHUDQJHZDVEHWZHHQWR\HDUVRIDJH$OO VWXGHQWVZHUHZHOOYHUVHGLQWKH(QJOLVKODQJXDJH$PDMRULW\RIWKHPZHUHQDWLYH(QJOLVK VSHDNHUV)LJXUHDQG)LJXUHSUHVHQWVRPHGHPRJUDSKLFVRQRXUSDUWLFLSDQWV        )LJXUH7KHQXPEHURISDUWLFLSDQWVVHOIUHSRUWLQJWKHSURJUDPPLQJODQJXDJHV WKH\NQHZRUDUHFXUUHQWO\OHDUQLQJ  )LJXUH<HDUVRISURJUDPPLQJH[SHULHQFHZLWKLQSDUWLFLSDQWV       7DVNV $WRWDORIWDVNV LQFOXGLQJWKHGHPRWDVN ZHUHJLYHQWRWKHWHVWSDUWLFLSDQWV 7KUHHRIWKHVHWDVNVZHUHDQDWXUDOWH[WSDUDJUDSK 17 (DFK17GLVFXVVHGDGLIIHUHQW WRSLFRQHLQYROYLQJWKHKLVWRU\RIEODFNSRZGHURQHLQYROYLQJWKHDGDSWDWLRQRIGXQJ EHHWOHVLQWRDQHZHQYLURQPHQWDQGRQHGLVFXVVLQJJRYHUQPHQWDQGHFRQRP\7KHRWKHU   WDVNVZHUHDOOVKRUW&VRXUFHFRGHSURJUDPV$OORIWKHSURJUDPVZHUHRIDQHDV\ HQRXJKGLIILFXOW\WKDWWKH,QWURGXFWLRQWR&RPSXWHU3URJUDPPLQJFODVVZRXOGKDYHEHHQ DEOHWRILJXUHRXWWKHRXWSXWRIWKHSURJUDPV7KHSURJUDPVUDQJHGIURPVLPSOHRXWSXW DQGLQSXWVWDWHPHQWVWRQHVWHGORRSVVRWKHUDQJHRIGLIILFXOW\GLGYDU\WKRXJKQRWWR LPSRVVLEOHVWDQGDUGV 7KHWHVWSDUWLFLSDQWVZHUHJLYHQDVPXFKWLPHDVWKH\QHHGHGWRUHDGWKH17DQG 6&WDVNV$FRPSUHKHQVLRQTXHVWLRQZDVJLYHQDIWHUHDFKWHVW7KHFRPSUHKHQVLRQ TXHVWLRQZDVUDQGRPO\FKRVHQEHWZHHQWKHIROORZLQJ$VXPPDU\RIWKH176&D PXOWLSOHFKRLFHRIWKH176&DIDFWDERXWWKH17RUWKHRXWSXWRIWKH6&7KHUHZDV DOVRDQRSWLRQIRUSDUWLFLSDQWVWRSXWWKDWWKH\GLGQ�WNQRZDQGZHUHHQFRXUDJHGQRWWR JXHVV7KHRUGHULQZKLFKWKHWDVNVZHUHGLVSOD\HGZDVGHWHUPLQHGE\DUDQGRPQXPEHU JHQHUDWRU(DFKFRPSUHKHQVLRQTXHVWLRQSHUWDVNZDVDOVRGHWHUPLQHGE\UXQQLQJD UDQGRPQXPEHUJHQHUDWRURYHUWKH17VILUVWDQGWKH6&VVHFRQG7KLVQRWRQO\JDYHXVDQ RYHUDOOXQGHUVWDQGLQJKRZWKHSDUWLFLSDQWVDQVZHUHGDVDZKROHEXWKHOSHGWRSUHYHQWWKH SDUWLFLSDQWVIURPGLVFXVVLQJWKHDQVZHUVRIWKHTXHVWLRQVZLWKHDFKRWKHU7DEOHDQG 7DEOHSUHVHQWWKHWDVNV     7DEOH2YHUYLHZRIVRXUFHFRGHXVHGLQWKHVWXG\7KHSURJUDPVZHUHPDUNHGDV6&WKURXJK 6&IRU&7KHGLIILFXOW\ZDVVHWEDVHGRQWKHW\SHVRIFRQVWUXFWVXVHG 6&,' 3URJUDP /2& &RQVWUXFWV 'LIILFXOW\ 6& &DNH3ULFHW[W  SVHXGRFRGHLIHOVH (DV\ 6& &DOFXODWH$YHUDJHFSS  $YHUDJHGLYLVLRQ 0HGLXP 6& &RXQW'RZQFSS  ZKLOHORRS (DV\ 6& 6LPSOH([SUHVVLRQFSS  2XWSXWVWDWHPHQW (DV\ 6& 6LJQ&KHFNHUFSS  ,IHOVHVWDWHPHQW 'LIILFXOW 6& 3ULQW3DWWHUQFSS  1HVWHGIRUORRS 'LIILFXOW 6& 'ROODUV$QG&HQWVFSS  GLYLVLRQDQGPRG 0HGLXP   7DEOH2YHUYLHZRIWKHQDWXUDOODQJXDJHWH[WVXVHGLQWKHVWXG\7KHSDUDJUDSKVZHUHPDUNHG DV17WKURXJK17 17,' 'HVFULSWLRQ 1XPEHURI:RUGV 17 5RFNHWV  17 %HHWOHV  17 *RYHUQPHQW   3OHDVHUHIHUWR$SSHQGL[$IRUGHWDLOVRQHDFKRIWKHVHVWLPXOL  'DWD&ROOHFWLRQDQG$SSDUDWXV 7KHH\HWUDFNHUXVHGLQWKHVWXG\ZDVWKH7RELL; ZZZWRELLFRP H\HWUDFNHU 7KHH\HWUDFNHULVD+]YLGHREDVHGELQRFXODUUHPRWHH\HWUDFNHUWKDWLVQRWLQWUXVLYH 7KHH\HWUDFNHUGRHVQRWUHTXLUHDKHDGSLHFHLWVLWVRQWKHGHVNGLUHFWO\LQIURQWRIWKH PRQLWRU$W+]WKHH\HWUDFNHULVDEOHWRJHQHUDWHVDPSOHVRIH\HGDWDSHUVHFRQG    7KH7RELL;KDVDQDFFXUDF\RIURXJKO\GHJUHHVRUDSSUR[LPDWHO\SL[HOV7KH H\HWUDFNHULVDEOHWRFRPSHQVDWHIRUKHDGPRYHPHQWGXULQJWKHVWXG\7KHVWXG\LWVHOI ZDVFRQGXFWHGRQDLQFKPRQLWRUZLWKD[UHVROXWLRQ:KLOHWKHUHZHUHWZR PRQLWRUVLQH[WHQGHGGHVNWRSVHWWLQJWKHWHVWSDUWLFLSDQWRQO\XVHGRQHPRQLWRUZKLOHWKH H[SHULPHQWHUXVHGWKHRWKHUWRVHWXSLQLWLDWHDQGPRQLWRUWKHVWXG\%HVLGHVWKHH\H WUDFNHUDXGLRDQGYLGHRRIWKHSDUWLFLSDQWVZHUHDOVRUHFRUGHGYLDD/RJLWHFKZHEFDP7KH H\HJD]HGDWDUHFRUGHGLQFOXGHGWLPHVWDPSVJD]HSRVLWLRQVIL[DWLRQVGXUDWLRQVSXSLOVL]H YDOLGLW\FRGHVDUHDVRILQWHUHVWVWDUWWLPHDQGHQGWLPHIRUHDFKWULDO,QWKLVVWXG\ZHXVH IL[DWLRQVGXUDWLRQVYDOLGLW\DQGDUHDRILQWHUHVW7KHDUHDVRILQWHUHVWZHUHJHQHUDWHG XVLQJH\H&RGH 0+DQVHQ 7KHHQWLUHVWXG\ZDVFRQGXFWHGLQ7RELL6WXGLR  6WXG\3URFHGXUHDQG,QVWUXPHQWDWLRQ 7KHVWXG\ZDVFRQGXFWHGLQDFFRUGDQFHZLWK,5%SROLFLHV ,5%3URWRFRO1XPEHU  DQGSURFHGXUHVDWWKH6(5(6/ 6RIWZDUH(QJLQHHULQJ5HVHDUFKDQG(PSLULFDO 6WXGLHV/DE RQWKH<RXQJVWRZQ6WDWH8QLYHUVLW\&DPSXV2QWKHGD\RIWKHVWXG\WHVW VXEMHFWVZHUHLQIRUPHGWKDWWKHSXUSRVHRIWKHVWXG\ZDVWRXQGHUVWDQGKRZSURJUDPPHUV UHDGERWKQDWXUDOODQJXDJHWH[WDQG&SURJUDPPLQJVRXUFHFRGH6XEMHFWVZHUHWKHQ DVNHGWRDQVZHUDSUHTXHVWLRQQDLUHRQOLQH(DFKSDUWLFLSDQWZDVLVVXHGDQ,'7KH VXEMHFWVZHUHVHDWHGDSSUR[LPDWHO\FPDZD\IURPWKHVFUHHQ%HIRUHWKHQDWXUDO WH[WWDVNVWKHXVHUZDVSUHVHQWHGZLWKDVFUHHQJLYLQJLQVWUXFWLRQVRQWKHXSFRPLQJWDVNV %HIRUHWKH&VRXUFHFRGHWDVNVWKHXVHUVZHUHSUHVHQWHGZLWKDVHWRILQVWUXFWLRQVDV ZHOODVDGHPRVRXUFHFRGHDORQJZLWKLWVTXHVWLRQVDQGWKHIRUPLQZKLFKWKH\VKRXOGEH DQVZHUHG7KHQDWXUDOWH[WWDVNVZHUHSUHVHQWHGILUVW LQDUDQGRPRUGHU IROORZHGE\WKH    VHYHQVRXUFHFRGHWDVNV LQDUDQGRPRUGHU 6XEMHFWVDQQRXQFHGRXWORXGZKHQWKH\ZHUH UHDG\WREHJLQWKHWHVWDQGXVHGWKHPRXVHWRFRQWLQXHRQWRWKHQH[WVOLGH6XEMHFWV VHOHFWHGRQHRIWKUHHQXPEHUV LQDUDQGRPRUGHU DIWHUHDFKWDVNDQGDQVZHUHGWKHTXHVWLRQ WKDWIROORZHGXVLQJWKHPRXVHRUNH\ERDUG$IWHUHDFKWDVNVXEMHFWVZHUHSUHVHQWHGZLWK DVKRUWTXHVWLRQQDLUHDERXWWKHGLIILFXOW\RIWKHWDVNDQGWKHLUFRQILGHQFHLQWKHLUDQVZHU $IWHUDOOWDVNVZHUHDQVZHUHGWHVWVXEMHFWVZHUHSUHVHQWHGZLWKDSRVWTXHVWLRQQDLUHWKDW DVNHGDERXWGLIILFXOW\WLPHQHHGHGDQGLIDQ\SUREOHPVRFFXUUHGGXULQJWKHWHVW        678'<5(68/76 :HSUHVHQWWKHUHVXOWVLQWHUPVRIRXUWZRUHVHDUFKTXHVWLRQV:HILUVWGHVFULEH KRZWKHGDWDZDVSURFHVVHGIROORZHGE\FRPSUHKHQVLRQWDVNDQGWLPLQJUHVXOWV7KUHDWV WRYDOLGLW\DQGDGLVFXVVLRQILQDOO\FORVHWKHFKDSWHU  3URFHVVLQJWKH'DWDIRU$QDO\VLV $IWHUWKHVWXG\ZDVFRPSOHWHWKHGDWDZDVH[SRUWHGIURP7RELL6WXGLRWRIODWILOHV %HIRUHZHUXQWKHVWDWLVWLFDOWHVWVRQWKHGDWDZHQHHGHGWRPDSWKHIL[DWLRQVRQVRXUFH FRGHHOHPHQWV:HXVHH\H&RGH 0+DQVHQ IRUWKLVSXUSRVHH\H&RGHLVDEOHWR DXWRPDWLFDOO\GHWHUPLQHOLQHVDQGZRUGVJLYHQDQLPDJHVWLPXOXV7KHVHOLQHVDQGZRUGV IRUPWKHDUHDVRILQWHUHVW $2, ,QRXUFDVHWKHLPDJHVWLPXOXVLVWKH17DQG6&WDVNV 7KHQLWDOVRPDSVWKHIL[DWLRQVRQFRUUHVSRQGLQJZRUGVVRZHDUHDEOHWRGHWHUPLQHZKLFK IL[DWLRQIDOOVRQZKLFKZRUGLQQDWXUDOWH[WRUVRXUFHFRGHHOHPHQW )LJXUHDQG)LJXUHVKRZXVWKHSLFWRULDOGHVFULSWLRQRIKRZH\H&RGHVHJPHQWV WKHOLQHVDQGZRUGV7KHIL[DWLRQVDUHQRWVKRZQLQWKHILJXUHVEXWWKH\DUHH[SRUWHGWRD ILOH             )LJXUH6RXUFHFRGH6& &DOFXODWH$YHUDJH ZLWKOLQH WRS DQGZRUG EHORZ  $2,V    VVVVVVVVVVVVVVVV  )LJXUH1DWXUDO/DQJXDJH17ZLWKOLQH WRS DQGZRUG EHORZ $2,V  %HIRUHZH SURFHHGWRDQVZHUWKHUHVHDUFKTXHVWLRQVZHILUVWUHSRUWRQWKH FRPSUHKHQVLRQDQGWLPLQJUHVXOWVRIWKHVWXG\:HXVHWKH0DQQ:KLWQH\WHVWWRFRPSDUH QRYLFHVDQGQRQQRYLFHV:HXVHWKH:LOFR[RQSDLUHGWHVWWRFRPSDUHZLWKLQHDFKJURXS    LHZLWKLQQRYLFHVRUZLWKLQQRQQRYLFHV7KHVHDUHQRQSDUDPHWULFWHVWVDVRXUVDPSOHLV QRWODUJHHQRXJKIRUSDUDPHWULFWHVWV  &RPSUHKHQVLRQ6FRUHV :HXVHWKHDEEUHYLDWLRQ17IRUQDWXUDOODQJXDJHWH[WVDQG6&IRUVRXUFHFRGH6HH )LJXUHDQG)LJXUHIRUFRPSUHKHQVLRQVFRUHV:HREVHUYHWKDWRYHUDOOQRQQRYLFHV VFRUHGKLJKHUWKDQQRYLFHV,Q)LJXUHZHFDQDOVRVHHWKHJDSEHWZHHQQRYLFHVDQGQRQ QRYLFHV SHU WDVN 7KH JDSV DUH ODUJHU IRU GLIILFXOW SURJUDPV OLNH 6LJQ&KHFNHU DQG 3ULQW3DWWHUQ7KHRWKHUSURJUDPVWKDWIHOOLQWRWKHPHGLXPDQGHDV\GLIILFXOW\FDWHJRU\KDG OHVVRIDJDSEHWZHHQQRYLFHVDQGQRQQRYLFHV7KLVLQGLFDWHVWKDWQRYLFHVKDGDKDUGWLPH JLYLQJDFRUUHFWDQVZHUIRUGLIILFXOWSUREOHPV      )LJXUH$YHUDJH&RPSUHKHQVLRQ6FRUHIRU1RYLFHVDQG1RQ1RYLFHVZLWKLQHDFK WDVN   )LJXUH$YHUDJH&RPSUHKHQVLRQ6FRUHIRU1RYLFHVDQG1RQ1RYLFHVDFURVVDOO WDVNV     7KH0DQQ:KLWQH\WHVWRQWKHWRWDOVFRUHVEHWZHHQQRYLFHVDQGQRQQRYLFHV VKRZHGVLJQLILFDQWGLIIHUHQFHV S 8  7KHQRQQRYLFHVGLGVLJQLILFDQWO\ EHWWHULHKLJKHUVFRUHVWKDQWKHQRYLFHV  7LPH7DNHQ )LJXUHWKURXJK)LJXUHVKRZVWKHWLPHWDNHQRQHDFKWDVNDQGRYHUDOOWDVNV FRPELQHG,WDOVRUHSRUWVRQWLPHGLIIHUHQFHVEHWZHHQQRYLFHVDQGQRQQRYLFHV%RWK QRYLFHVDQGQRQQRYLFHVVSHQWWKHOHDVWDPRXQWRIWLPHLQ6&ZKLFKZDVDVPDOODQG VLPSOHSURJUDP6LPLODUWRFRPSUHKHQVLRQVFRUHVWKHJDSVEHWZHHQQRYLFHVDQGQRQ QRYLFHVDUHPXFKPRUHDSSDUHQWLQGLIILFXOWSURJUDPV1RQQRYLFHVDOVRWHQGWRWDNHWKH VWXG\PRUHVHULRXVO\FRPSDUHGWRQRYLFHVDQGVRWKH\VSHQGPRUHWLPHRQWKHWDVNV$ 0DQQ:KLWQH\WHVWUHYHDOVQRVLJQLILFDQWGLIIHUHQFHVLQWLPHEHWZHHQQRYLFHVDQGQRQ QRYLFHV S 8        )LJXUH$YHUDJHWRWDOWLPHVSHQWE\QRYLFHVRQHDFKWDVN   )LJXUH$YHUDJHWRWDOWLPHVSHQWE\QRQQRYLFHVRQHDFKWDVN      )LJXUH$YHUDJHWRWDOWLPHVSHQWRQHDFKWDVNEHWZHHQQRYLFHVDQGQRQQRYLFHV   )LJXUH$YHUDJHWRWDOWLPHVSHQWRQDOOWDVNV      54,VWKHUHDQLQKHUHQWGLIIHUHQFHLQWKHZD\QRYLFHSURJUDPPHUVUHDG QDWXUDOODQJXDJHWH[WFRPSDUHGWRVRXUFHFRGH" ,QRUGHUWRDQVZHU54ZHUHSRUWRQWKHOLQHDULW\PHWULFVZHL QWURGXFHGLQ6HFWLRQ )LJXUHVKRZVWKHILYHOLQHDULW\PHWULFVIRU17DQG6&1RWLFHWKDWWKHPHDVXUHV 9HUWLFDO1H[W7H[W9HUWLFDO/DWHU7H[WDQG+RUL]RQWDO/DWHU7H[WDUHDOOKLJKHUIRU17 FRPSDUHGWR6&7KHVHDUHDOOOLQHDULW\PHDVXUHVLQGLFDWLQJWKDW17LVUHDGPRUHOLQHDUO\ WKDQ6&7KHUHJUHVVLRQPHDVXUHVGHDOZLWKQRQOLQHDUUHDGLQJ7KHOLQHUHJUHVVLRQUDWHV ZHUHKLJKHULQ17WKDQ6&LHSDUWLFLSDQWVZHQWEDFNWRUHDGDOLQHPRUHRIWHQLQ17WKDQ 6& :HUDQWKH:LOFR[RQSDLUHGWHVWZLWKLQQRYLFHVWRVHHLIWKHDERYHPHDVXUHVZHUH VLJQLILFDQWO\GLIIHUHQW7DEOHVKRZVWKHUHVXOWVRIWKH:LOFR[RQWHVW:HVHHWKDWWKH PHDVXUHV9HUWLFDO1H[W7H[W9HUWLFDO/DWHU7H[WDQG+RUL]RQWDO/DWHU7H[WDQG/LQH 5HJUHVVLRQ5DWHDUHDOOVLJQLILFDQWO\GLIIHUHQWEHWZHHQ17DQG6&7KH5HJUHVVLRQ5DWH ZKLFKLQYROYHVUHJUHVVLRQVRIDQ\OHQJWKLVQRW7KLVPHDQVWKDWWKHUHLVVLJQLILFDQWPRUH OLQHUHJUHVVLRQVLQ6&YV17UHDGLQJ7KLVDOVRLQGLFDWHVWKDW6&LVKDUGHUIRUQRYLFHVWR UHDGWKDQ17HYHQWKRXJKWKHUHDGLQJLVOHVVOLQHDU             )LJXUH$YHUDJHOLQHDULW\PHDVXUHVIRUDOOQRYLFHV     7DEOH:LOFR[RQVLJQHGUDQNHGWHVWIRU17YV6&IRU1RYLFHV /LQHDULW\0HDVXUH 6XPRI3RVLWLYH6LJQHG 5DQNV p 9HUWLFDO1H[W7H[W  p   9HUWLFDO/DWHU7H[W  p   +RUL]RQWDO/DWHU7H[W  p   5HJUHVVLRQ5DWH  p  /LQH5HJUHVVLRQ5DWH  p    ,Q)LJXUHZHVHHWKDWWKHOHQJWKRIWKHVDFFDGHLVKLJKHUIRU17WKDQ6&7KLV PHDQVWKDWLQ17WKH\MXPSHGDIHZOLQHVPRUHWKDQLQWKH6&,Q)LJXUHZHSUHVHQW WKHHOHPHQWFRYHUDJH7KLVPHDQVWKDWRI17LQQRYLFHVZHUHORRNHGDWDQG RIHOHPHQWVZHUHORRNHGDWIRU6&7KLVPHDVXUHVLPSO\LQGLFDWHVKRZPDQ\HOHPHQWVRQ WKHVWLPXOXVZHUHORRNHGDW:HDOVRGLVFXVVWKLVSRLQWLQWKHWKUHDWVWRYDOLGLW\VHFWLRQ  )LJXUH6DFFDGHOHQJWKIRUDOOQRYLFHVLQ17DQG6&      )LJXUH(OHPHQWFRYHUDJHIRUDOOQRYLFHVLQ17DQG6&  %RWKVDFFDGHOHQJWK p  DQGHOHPHQWFRYHUDJH p  DUHVLJQLILFDQWO\ GLIIHUHQWEHWZHHQ17DQG6&7KLVLVDQRWKHULQGLFDWLRQWKDW17DQG6&DUHTXLWHGLIIHUHQW LQWHUPVRIWKHLUFRJQLWLYHORDG :HQRZGLVFXVVWKHDOLJQPHQWRI17DQG6&WR6WRU\2UGHUDQGVKRZWKHUHVXOWVRIWKH 1HHGOHPDQ:XQFK 1:DOJRULWKP LQ7DEOH6WRU\RUGHULVEDVLFDOO\UHDGLQJWKH VWLPXOXVRQHOLQHDWDWLPHIURPWRSWRERWWRP W\SLFDOO\WKHZD\ZHUHDGQDWXUDOODQJXDJH WH[W 7KH1:DOJRULWKPLVXVHGDVDVWULQJPDWFKLQJDOJRULWKPWRGHWHUPLQHVWRU\RUGHU ,WKDVDOVREHHQXVHGE\&ULVWLQRHWDO &ULVWLQRHWDO LQHDUOLHUZRUNRQH\H PRYHPHQWUHVHDUFK7KHDOJRULWKPJLYHVDVLPLODULW\VFRUHZKHUHDKLJKVFRUHLQGLFDWHV WKDWWZRVHTXHQFHVDUHFORVHWRHDFKRWKHU7KHGLIIHUHQFHEHWZHHQQDwYHDQGG\QDPLF VFRUHVLVZKHWKHUUHSHWLWLRQVWKURXJKWKHFRGHDUHFRXQWHG6RLIZHFDUHDERXWKRZPDQ\ WLPHVWKHSHUVRQUHDGWKURXJKWKHFRGHZHNHHSUHSHDWLQJWKHVWULQJPDWFKLQJZLWKWKH VWRU\RUGHUDQGWKHH\HJD]HPRYHPHQWVWRJHWDG\QDPLFVFRUH     7DEOH1HHGOHPDQ:XQFK5HVXOWVFRPSDULQJWKHVWRU\RUGHUIRU17DQG6&IRU QRYLFHV 6WRU\2UGHU 17 6& S 1DwYH1:6FRUH r8.27r21.16 <0.001 '\QDPLF1:6FRUH  18.71r4.06 <0.001 5HSHWLWLRQV 3.42 2.6 <0.001  7KHVHVFRUHVDUHQRWFORVHWRRQHDQRWKHU7KLVPHDQVWKDWHYHQQRYLFHVGRQRWVWDUW ZLWKDQDSSURDFKWKDWLVYHU\WRSGRZQDQGOHIWWRULJKW FRQWUDU\WRZKDW %XVMDKQHWDO  IRXQG 7KHUHVXOWVLQGLFDWHWKDWERWKQDWXUDOODQJXDJHWH[WDQGVRXUFHFRGHZHUH ERWKUHDGPXOWLSOHWLPHV7KH17ZDVUHDGWLPHVFRPSDUHGWR6&ZKLFKZDVUHDG WLPHV7KHPRUHUHDGWKURXJKVWKHKLJKHUWKH1:DOLJQPHQWVFRUH:HIRXQGDVLJQLILFDQW GLIIHUHQFHEHWZHHQ17DQG6&�VVWRU\RUGHUIRUQRYLFHV  1RQQRYLFHSURJUDPPHUV1DWXUDOODQJXDJHWH[WYVVRXUFHFRGH 6HH)LJXUHIRUDVLPLODUVHWRIOLQHDULW\PHWULFVIRUQRQQRYLFHV VLPLODUWRZKDW ZHVKRZHGIRUQRYLFHVLQWKHSUHYLRXVVHFWLRQ,Q7DEOHZHGRDQHTXLYDOHQWDQDO\VLVRQ QRQQRYLFHVE\UXQQLQJWKH:LOFR[RQWHVWWRGHWHFWIRUGLIIHUHQFHVLQKRZQRQQRYLFHVUHDG 17YV6&,WIROORZVWKHVDPHJHQHUDOWUHQGDVQRYLFHVZLWKRQ HH[FHSWLRQ7KHUHJUHVVLRQ UDWHLVDOVRVLJQLILFDQWO\GLIIHUHQWIRUH[SHUWVEHWZHHQ17DQG6&          )LJXUH$YHUDJHOLQHDULW\PHDVXUHVIRUDOOQRQQRYLFHV     7DEOH:LOFR[RQVLJQHGUDQNHGWHVWIRU17YV6&IRUQRQQRYLFHV  /LQHDULW\0HDVXUH 6XPRI3RVLWLYH6LJQHG 5DQNV p 9HUWLFDO1H[W7H[W  p   9HUWLFDO/DWHU7H[W  p   +RUL]RQWDO/DWHU7H[W  p   5HJUHVVLRQ5DWH  p   /LQH5HJUHVVLRQ5DWH  p    )LJXUHDQG)LJXUHVKRZWKHVDFFDGHOHQJWKDQGHOHPHQWFRYHUDJHUHVSHFWLYHO\ IRUQRQQRYLFHV%RWKWKHVDFFDGHOHQJWK S  DQGFRYHUDJH S  ZHUH VLJQLILFDQWO\GLIIHUHQWIRU17DQG6&LQWKHQRQQRYLFHVJURXSDVZHOO  )LJXUH6DFFDGHOHQJWKIRUDOOQRQQRYLFHVLQ17DQG6&      )LJXUH(OHPHQWFRYHUDJHIRUDOOQRQQRYLFHVLQ17DQG6&   54'RHVH[SHUWLVHSOD\DUROHLQWKHUHDGLQJEHKDYLRURISURJUDPPHUVLQ SDUWLFXODUZLWKUHVSHFWWROLQHDUUHDGLQJ" :HUDQWKH0DQQ:KLWQH\WHVWRQDOOQRYLFHVYVQRQQRYLFHV:HGLGQRWILQGDQ\ VLJQLILFDQWGLIIHUHQFHVLQQRYLFHVYVQRQQRYLFHVKHUHDVZDVIRXQGLQ %XVMDKQHWDO  ,WFRXOGEHEHFDXVHRXUQRQQRYLFHVZHUHQRWUHDOO\H[SHUWVLQLQGXVWU\7KH\ZHUH MXVWVOLJKWO\PRUHDGYDQFHGWKDQQRYLFHV7DEOHVKRZVWKHUHVXOWVRIWKH0DQQ:KLWQH\ WHVW          7DEOH0DQQ:KLWQH\UHVXOWVIRUQRYLFHVYVQRQQRYLFHVRYHUD OOWDVNV /LQHDULW\0HDVXUH 8 p 9HUWLFDO1H[W7H[W  p  9HUWLFDO/DWHU7H[W  P  +RUL]RQWDO/DWHU7H[W  p  5HJUHVVLRQ5DWH  p  /LQH5HJUHVVLRQ5DWH  p   :HSODQRQFRQGXFWLQJWKHVWXG\ZLWKUHDOH[SHUWVWRVHHLIDGLIIHUHQFHFDQEH IRXQGIRUOLQHDULW\PHWULFV  3RVW4XHVWLRQQDLUH5HVXOWVDQG4XRWHV $IWHUWKHVWXG\ZHDVNHGWKHSDUWLFLSDQWVLIWKH\KDGHQRXJKWLPHWRFRPSOHWHWKH WDVNVZKLFKZDVXQDQLPRXVO\DJUHHGXSRQDVD\HV3DUWLFLSDQWVZHUHDOVRDVNHGDERXWWKH GLIILFXOWRIWKHVWXG\RYHUDOOOHVVWKDQIRXQGLWYHU\HDV\IRXQGLWHDV\ FRQVLGHUHGLWDYHUDJHDQGIRXQGLWGLIILFXOW7KHODVWTXHVWLRQDVNHGZDVDERXWDQ\ GLIILFXOW\IDFHGGXULQJWKHVWXG\SDUWLFLSDQWVDJUHHGWKDWLWZDVGLIILFXOWZKHQLWFDPHWR UHPHPEHUWKHVWLPXOLDQGDQVZHUWKHFRPSUHKHQVLRQTXHVWLRQ7KHIROORZLQJWDEOHJLYHV VRPHLQVLJKWDVWRZKDWVRPHSDUWLFLSDQWVWKRXJKWRIWKHVWXG\RYHUDOO       7DEOH3RVW4XHVWLRQQDLUHRSLQLRQV ,' 'LG\RXKDYH VXIILFLHQWWLPH WRFRPSOHWHWKH VWXG\" +RZZRXOG\RX UDWHWKHRYHUDOO GLIILFXOW\RIWKH WDVNV" 'HVFULEHDQ\GLIILFXOW\\RXIDFHG GXULQJWKHVWXG\ 3 <HV 'LIILFXOW ,IDFHGVRPHGLIILFXOW\ZLWKWKHSURJUDPV EHFDXVH,GLGQRWXQGHUVWDQGSDUWVRILW 3 <HV (DV\ 7KHKDUGHVWSDUWZDVPHPRUL]LQJZKDW WKHRXWSXWZDVDVRSSRVHGWRILJXULQJRXW ZKDWWKHSURJUDPZDVGRLQJ 3 <HV 'LIILFXOW XQGHUVWDQGLQJ ZKDW LW LV DVNLQJ DQG UHPHPEHULQJLW 3 <HV $YHUDJH 5HPHPEHULQJ ZKDW ZDV RQ SUHYLRXV VFUHHQVZDVGLIILFXOW 3 <HV 9HU\HDV\ ,GLGQ WUHPHPEHUWKRVHUHDGLQJV 3 <HV (DV\ LPD\QRWKDYHDVPXFKSURJUDPPLQJ H[SHULHQFHDVLZRXOGOLNHWRKDYHKDGWR DQVZHUWKHVHTXHVWLRQV 3 <HV 9HU\HDV\ 6KRUWWHUPPHPRU\ORVV 3 <HV $YHUDJH , KDG QR UHDO GLIILFXOW\ ZLWK WKH GLUHFWLRQVRUKRZWRFRPSOHWHWKHVWXG\ 7KHRQO\UHDOSUREOHP,IDFHGZDVWKH RXWSXWRIWKHLQFUHPHQWDOSURJUDP7KDW ZDVEHFDXVHRIP\RZQVNLOOOHYHOQRW WKHSURJUDPLWVHOI  7KHUHVXOWVRIWKHVWXG\LQGLFDWHWKDWQRYLFHVDQGQRQQRYLFHVERWKUHDGVRXUFHFRGH VLPLODUO\KRZHYHUQRQQRYLFHVUHDGFRGHOHVVOLQHDUO\WKDQQDWXUDOODQJXDJH$OVRWKH OHYHORIWKHSURJUDPPHUFKDQJHVWKHZD\WKDWWKH\VHHDSURJUDP:HDVNHGSDUWLFLSDQWV DERXWZKDWZDVJRLQJRQLQWKHLUPLQGVDVWKH\ZDWFKHGWKHLUH\HPRYHPHQWVIRUWZR GLIIHUHQWSURJUDPV3ULQW3DWWHUQ7HVWDQG6LJQ&KHFNHU7DEOHVKRZVRPHRIWKHVHTXRWHV         7DEOH7KRXJKW3URFHVV4XRWHV 1DPH 6NLOO/HYHO 3URJUDP 4XRWH 3 1RYLFH 3ULQW3DWWHUQ I was trying to do the loop, just go over it trying to make a tracing table   6LJQ&KHFNHU We haven't gone over strings yet so I figured it was just like everything else. I kind of forgot to look at the output. 3 1RQ QRYLFH 3ULQW3DWWHUQ I noticed I'm skipping all over the place, I was reading real fast skimming, but kept bouncing up to the top to know what I was thinking   6LJQ&KHFNHU I was sort of just reading what it was going to do and I ended up having to pop back up to the top to see how many i and j were because I wasn't exactly paying attention to variables at the start 3 1RYLFH 3ULQW3DWWHUQ Obviously I have no clue what the computer programming is and I knew I couldn't go back to it so a lot of it was trying to remember the little things.   6LJQ&KHFNHU I had no idea, this was the first one I literally had no idea what was going on, absolutely nothing and I was just looking for patterns. That the number was 0 and what the number means. You can kind of understand it, most of it was just trying to remember if i got asked if the number was greater than 0 or less than 0 it's negative. I got what it meant really fast I just wanted to make sure, yeah.      7KUHDWVWR9DOLGLW\ :HQRZGLVFXVVWKHWKUHDWVWRYDOLGLW\WRWKLVVWXG\7RDFFRXQWIRUGLIIHUHQWFRQWURO VWUXFWXUHVLQVRXUFHFRGHDVZHOODVGLIIHUHQWZRUGOHQJWKVLQQDWXUDOODQJXDJHWH[WZHXVHG WKUHH17SDVVDJHVDQGVHYHQVRXUFHFRGHSDVVDJHV7KHIDFWWKDWZHGLGQRWILQGDQ\ GLIIHUHQFHVEHWZHHQQRYLFHVDQGQRQQRYLFHVLQGLFDWHVWKDWWKH\DUHSRVVLEO\DWWKHVDPH OHYHORIUHDGLQJVNLOOV,QRUGHUWRILQGGLIIHUHQFHVLQOLQHDULW\LWPLJKWEHQHFHVVDU\WR VWXG\H[SHUWSURJUDPPHUVLQLQGXVWU\ZKRSURJUDPRQDGDLO\EDVLVDQGKDYHEHHQZRUNLQJ LQLQGXVWU\IRUPRUHWKDQ\HDUV 2QHPDMRUWKUHDWWRYDOLGLW\LVWKHVNHZQHVVWKDWRFFXUVLQH\HWUDFNLQJGDWD7KH OLQHDULW\PHWULFVDUHGLUHFWO\GHSHQGHQWRQKRZDFFXUDWHO\WKHIL[DWLRQVDUHPDSSHGWR ZRUGVRUVRXUFHFRGHHOHPHQWV:HGLGQRWFRUUHFWVNHZQHVVIRUWKLVVWXG\DQGKHQFHWKH UHVXOWVVKRXOGEHWDNHQZLWKFDXWLRQ:HDOVRGLVFDUGHGDOOWULDOVZLWKOHVVWKDQ PDSSLQJRQVRXUFHFRGHHOHPHQWVDQGIRXQGWKHVDPHVLJQLILFDQWUHVXOWV7KLVLQGLFDWHV WKDWZHPLJKWILQGHYHQPRUHVWUHQJWKDQGHIIHFWLQRXUILQGLQJVLIDOOWKHGDWDZDV FRUUHFWHG:HDUHFXUUHQWO\ZRUNLQJRQDQDXWRPDWHGIL[DWLRQFRUUHFWLRQDOJRULWKP $OVRRQH[DPLQLQJWKHVFDQSDWKVZHIRXQGWKDWPRVWRIWKHPZHUHFORVHWRWKH ZRUGWKDWWKH\ZHUHORRNLQJDW:HVWURQJO\EHOLHYHWKDWDIWHUFRUUHFWLRQZHVKRXOGVHH HYHQVWURQJHUVLJQLILFDQFH:HKDYHOHIWWKLVDVDQLPPHGLDWHIXWXUHH[HUFLVH     &21&/86,216$1')8785(:25. :HFRQGXFWHGDVWXG\WKDWFKDUDFWHUL]HGOLQHDULW\LQH\HPRYHPHQWVEHWZHHQ QDWXUDOODQJXDJHWH[WDQGVRXUFHFRGHLQ&7KLVVWXG\UHSOLFDWHVDQHDUOLHUVWXG\WKDW ORRNVLQWR-DYDFRGH2XUUHVXOWVVKRZWKDWERWKQRQQRYLFHVDQGQRYLFHVUHDGVRXUFHFRGH VLJQLILFDQWO\GLIIHUHQWWKDQQDWXUDOODQJXDJHWH[WZKLOHPRVWQDWXUDOWH[WLVUHDGOHIWWR ULJKWWRSWRERWWRPZLWKIHZUHJUHVVLRQVVRXUFHFRGHLVUHDGLQDOHVVOLQHDUPDQQHUZLWK PRUHUHJUHVVLRQV:KLOHWKHVHILQGLQJVDUHGLIIHUHQWDJJUHJDW HGDWDVKRZVWKDWWKHUHLVQR GLIIHUHQFHRYHUDOOEHWZHHQQRYLFHVDQGQRQQRYLFHVLQRXUVWXG\VDPSOH:HFRQVLGHUWKLV VWXG\DVDILUVWRIWZRVWXGLHVGRQHWKLV\HDU  WKDWTXDQWLILHVOLQHDULW\(YHQWKRXJK WKLVVHHPVWREHTXLWHLQWXLWLYHWKHUHZDVQRSURRIWKDWWKLVZDVLQGHHGWKHFDVH $VSDUWRIIXWXUHZRUNZHDUHFXUUHQWO\FRQGXFWLQJDVHFRQGSKDVHRIWKLVVWXG\ ZLWKWKHVDPHJURXSRIVWXGHQWV7KHSXUSRVHLVWRGHWHUPLQHLIWKHH\HPRYHPHQWVGLIIHU DWWKHHQGRIWKHVHPHVWHULQGLFDWLQJLIDQ\OHDUQLQJRFFXUUHG7KLVSKDVHRIWKHVWXG\LV EHLQJFRQGXFWHGGXULQJWKHODVWZHHNRIWKHVHPHVWHU:HDUHDOVRWDNLQJDORRNDWIL[DWLRQV DQGGXUDWLRQVRQVSHFLILFDUHDV EHDFRQV LQWKHFRGHSURYLGHGDQGZDQWWRGHWHUPLQHLIWKH GLIILFXOW\RIDWDVNPDNHVDGLIIHUHQFHRQKRZERWKQRYLFHVDQGH[SHUWVUHDGWKHFRGH %HDFRQVDUHSODFHVLQWKHFRGHWKDWQRQQRYLFHVWHQGWRIRFXVRQDVRQHFKXQNRIGDWD:H EHOLHYHWKDWVWXGLHVOLNHWKLVRQHKHOSWRZDUGVTXDQWLI\LQJH\HPRYHPHQWVIRUVSHFLILF WDVNV,QRXUFDVHWKHWDVNZDVWRTXDQWLI\OLQHDULW\0RUHVWXGLHVDQGUHSOLFDWLRQVQHHGWR EHGRQHWRDGGWRWKHERG\RINQRZOHGJHDQGWKHUHE\DGYDQFHWKHVWDWHRIWKHDUWLQ H[SHULPHQWDWLRQ    $33(1',;$6WXG\0DWHULDO5HSOLFDWLRQ3DFNDJH $6WXG\,QVWUXFWLRQV 7KHSXUSRVHRIWKLVVWXG\LVWRXQGHUVWDQGKRZSHRSOHFRPSUHKHQGFRGHDQGQDWXUDO ODQJXDJHV,WLVQRWWRPHDVXUH\RXUVNLOOVDWSURJUDPPLQJ7KHUHDUHWZRSDUWVWRWKLVVWXG\ )LUVW\RXZLOOEHDVNHGWRUHDGWKUHH(QJOLVKSDVVDJHVDQGDQVZHUDTXHVWLRQDERXWWKHP 7KHVHFRQGSDUWRIWKHVWXG\ZLOOLQYROYHUHDGLQJ&FRGHDQGDQVZHULQJDTXHVWLRQDERXW WKHFRGH 7KUHHSRVVLEOHFRPSUHKHQVLRQTXHVWLRQV\RXFRXOGEHDVNHGDUH $PXOWLSOHFKRLFHTXHVWLRQDERXWWKHDOJRULWKPLFLGHD RUWH[WSDVVDJHIRUWKH SDVVDJHV  25 :KDWLVWKHRXWSXWRIWKHSURJUDP" VRPHIDFWDERXWWKHSDVVDJHV  25 *LYHDVXPPDU\RIWKHSURJUDP RUSDVVDJHV +HUH\RXZLOOVD\ZKDWWKHSURJUDP SDVVDJH GRHVDQG ZKDWWKHDOJRULWKPLFLGHDLV  x <RXPD\VWXG\WKH(QJOLVKWH[WDQGFRGHVQLSSHWIRUDVORQJDV\RXOLNH%XWDV VRRQDV\RXDUHGRQHXQGHUVWDQGLQJZKDWLWGRHVZHDVNWKDW\RXPRYHRQWRDQVZHU WKHTXHVWLRQIROORZLQJLW x 3OHDVHGRQRWJXHVVWKHDQVZHUV x 7KHVRXUFHFRGHJLYHQWR\RXGRHVQRWFRQWDLQDQ\EXJV x ,ZRXOGOLNHWRHQFRXUDJH\RXWRSOHDVHWKLQNDORXG9HUEDOL]H\RXUWKRXJKWVDVWKLV ZLOOKHOSXVXQGHUVWDQG\RXUWKRXJKWSURFHVV x 3OHDVHDQVZHUWKHTXHVWLRQVIURPWKHSHUVSHFWLYHRIDSHUVRQWU\LQJWRXQGHUVWDQG DQGFRPSUHKHQGWKHFRGH x <RXZLOOILOOLQ\RXUDQVZHUVLQDZHEIRUPWKDWZLOOSRSXSDXWRPDWLFDOO\ x )RUHDFKTXHVWLRQ\RXZLOOEHDVNHGWRUDWHWKHGLIILFXOW\OHYHO\RXIDFHGDQG\RXU FRQILGHQFHLQWKHDQVZHU\RXSURYLGHG8VHWKHPRXVHWRVHOHFWWKHRSWLRQV x :KHQ\RXDUHGRQHZLWKWKHUHDGLQJWKHFRGHRQWKHVFUHHQFOLFNWKH/()7PRXVH EXWWRQRQO\RQFHWRDGYDQFHWRWKHQH[WVFUHHQ x ,PSRUWDQWQRWH<RXZLOOQRWEHDEOHWRVHHWKHVRXUFHFRGHZKLOH\RXDUHDQVZHULQJ WKHTXHVWLRQ x 7KHUHLVQRSRVVLELOLW\WRJREDFN3OHDVHEHVXUHRI\RXUFKRLFHVEHIRUH\RX DGYDQFHWRWKHQH[WVFUHHQ      x 3OHDVHWU\WRPDLQWDLQ\RXUSRVLWLRQLQWKHFKDLUZKLOH\RXGRWKHVWXG\VRWKDWZH GRQRWORVHWKHWUDFNLQJRI\RXUH\HV0RYLQJWKHFKDLUEDFNRUPRYLQJ\RXUVHOI EDFNLQWKHFKDLUZLOOFDXVHWKHH\HWUDFNHUWRVWRSWUDFNLQJ6PDOOKHDGPRYHPHQWV VXFKDVORRNLQJDWWKHNH\ERDUGWRW\SHVKRXOGEHILQH x )LQGDFRPIRUWDEOHSRVLWLRQVRZHFDQEHJLQ x :HZLOOILUVWEHJLQZLWKFDOLEUDWLQJ\RXUH\HV/RRNDWWKHEODFNGRWLQWKHFHQWHU RIWKHUHGFLUFOHDQGIROORZLWDURXQGRQWKHVFUHHQ x 7KHQZHZLOOEHJLQZLWKDVDPSOHWDVNIROORZHGE\WKHDFWXDOVWXG\TXHVWLRQV  :HUHTXHVWWKDW\RXSOHDVHQRWOHWVWXGHQWVZKRKDYHQRWWDNHQWKHVWXG\LQ\RXUFODVVNQRZ RIWKHTXHVWLRQVDVNHGDIWHU\RXOHDYH7KDQN\RXIRUPDLQWDLQLQJWKHLQWHJULW\RIWKHGDWD  :HDSSUHFLDWH\RXUWLPHLQWDNLQJWKLVVWXG\     $3UH4XHVWLRQQDLUH ,'   :KDWLV\RXUQDWLYHO DQJXDJH"   <RXU(QJOLVK/HYHO"  o 1RQH o /RZ o 0HGLXP o +LJK <RXURYHUDOOSURJUDPPLQJH[SHUWLVH"  o 1RQH o /RZ o 0HGLXP o +LJK <RXUH[SHUWLVHLQ&"  o 1RQH o /RZ o 0HGLXP o +LJK $JH  o \HDUV o �\HDUV o �\HDUV o !\HDUV *HQGHU  o 0DOH o )HPDOH <HDUVRISURJUDPPLQJH[SHULHQFHLQ&  o 1RQH o  o \HDUV o \HDUV o !\HDUV +RZRIWHQGR\RXSURJUDPLQ&"  o OHVVWKDQKRXUPRQWK o OHVVWKDQKRXUZHHN o OHVVWKDQKRXUGD\ o PRUHWKDQKRXUVGD\  <HDUVRISURJUDPPLQJH[SHULHQFHLQDQ\ODQJXDJH     o 1RQH o  o \HDUV o \HDUV o !\HDUV +RZRIWHQGR\RXSURJUDPLQDODQJXDJHRWKHUWKDQ&"  o OHVVWKDQKRXUPRQWK o OHVVWKDQKRXUZHHN o OHVVWKDQKRXUGD\ o PRUHWKDQKRXUVGD\ :KHQGLG\RXILUVWVWDUWSURJUDPPLQJ" OLVWWKH\HDU    :KLFKODQJXDJHGLG\RXILUVWOHDUQSURJUDPPLQJLQ"   :KDWSURJUDPPLQJODQJXDJHFODVVHVDUH\RXFXUUHQWO\WDNLQJ"  6WDWHWKHFRXUVHQXPEHUIRUHJ&6,6DQGRU&6,6  :KLFKSURJUDPPLQJODQJXDJHVGR\RXNQRZ"  6WDWHWKHODQJXDJHDQGOHYHORIH[SHUWLVH ORZPHGLXPKLJK   'R\RXZDQWWRVD\DQ\WKLQJHOVHDERXW\RXUSURJUDPPLQJEDFNJURXQG",I\HV\RXPD\ HQWHULWEHORZ    $7KH6WLPXOL1DWXUDO7H[W 17 DQG6RXUFH&RGH 6&  17&LYLOLVDWLRQ   17%HHWOHV     17*RYHUQPHQW  'HPR6RXUFH&RGHDQG6ROXWLRQ6KRZQ      6&&DNHSULFHSVHXGRFRGH           6&&DOFXODWH$YHUDJH      6&&RXQW'RZQ     6&6LPSOH([SUHVVLRQ           6&6LJQ&KHFNHU      6&'ROODUVDQG&HQWV        6&3ULQW3DWWHUQ7HVW       $7KH&RPSUHKHQVLRQ4XHVWLRQVIRUWKH17DQG6&6WLPXOL %HIRUHHDFKWH[WRQHRIWKHVHWKUHHLQVWUXFWLRQVZLOOEH JLYHQ  3OHDVHUHDGDQGFRPSUHKHQGWKHIROORZLQJWH[W:KHQ\RXDUH GRQHSUHVVWKHOHIWPRXVHEXWWRQ 7KHQ\RXZLOOEHJLYHQD08/7,3/(&+2,&(TXHVWLRQDERXWWKH FRQWHQW  3OHDVHUHDGDQGFRPSUHKHQGWKHIROORZLQJWH[W:KHQ\RXDUH GRQHSUHVVWKHOHIWPRXVHEXWWRQ 7KHQ\RXZLOOEHDVNHGDERXW620()$&7IURPWKHWH[WV  3OHDVHUHDGDQGFRPSUHKHQGWKHIROORZLQJWH[W:KHQ\RXDUH GRQHSUHVVWKHOHIWPRXVHEXWWRQ 7KHQ\RXZLOOEHDVNHGWRJLYHD6800$5<  <RXZLOOQRWNQRZZKLFKRIWKHDERYHTXHVWLRQV PXOWLSOHFKRLFHVRPHIDFWRU VXPPDU\ \RXZLOOEHDVNHGDERXW  <RXZLOOQRWEHDEOHWRYLHZWKHSDVVDJHZKLOH\RXDQVZHUWKHTXHVWLRQV  6D\%HJLQZKHQ\RXDUHUHDG\WRVWDUW                                                                              $3RVW4XHVWLRQQDLUH  ,'  o 7KLVLVDUHTXLUHGTXHVWLRQ 'LG\RXKDYHVXIILFLHQWWLPHWRFRPSOHWHWKHVWXG\"  o <HV o 1R o 7KLVLVDUHTXLUHGTXHVWLRQ +RZZRXOG\RXUDWHWKHRYHUDOOGLIILFXOW\RIWKHWDVNV"  o 9HU\HDV\ o (DV\ o $YHUDJH o 'LIILFXOW o 9HU\'LIILFXOW 'HVFULEHDQ\GLIILFXOW\\RXIDFHGGXULQJWKHVWXG\ $Q\RWKHUFRPPHQWVDERXWWKHVWXG\DUHZHOFRPH     5HIHUHQFHV $OL16KDUDIL=*XpKpQHXF<* $QWRQLRO*  $QHPSLULFDOVWXG\RQWKH LPSRUWDQFH RI VRXUFH FRGH HQWLWLHV IRU UHTXLUHPHQWV WUDFHDELOLW\ Empirical Software Engineering�KWWSGRLRUJV\ %DQNHU5''DYLV*% 6ODXJKWHU6$  6RIWZDUH'HYHORSPHQW3UDFWLFHV 6RIWZDUH&RPSOH[LW\DQG6RIWZDUH0DLQWHQDQFH3HUIRUPDQFH$)LHOG6WXG\ Management Science44  �KWWSGRLRUJPQVF %HGQDULN5 7XNLDLQHQ0  7HPSRUDO(\H7UDFNLQJ'DWD �(YROXWLRQRI 'HEXJJLQJ6WUDWHJLHVZLWK0XOWLSOH5HSUHVHQWDWLRQV&DVH6WXG\ �9LVXDO$WWHQWLRQ 'XULQJ'HEXJ,Q2008 Symposium on Eye Tracking Research & Applications (Savannah, Georgia, March 26 - 28, 2008) 9ROSS� 1HZ<RUN(75$ �$&0 %LQNOH\''DYLV0/DZULH'0DOHWLF-,0RUUHOO& 6KDULI%  7KH LPSDFW RI LGHQWLILHU VW\OH RQ HIIRUW DQG FRPSUHKHQVLRQ Empirical Software Engineering18  �KWWSGRLRUJV %URRNV 5   $ 7KHRUHWLFDO $QDO\VLV RI WKH 5ROH RI 'RFXPHQWDWLRQ LQ WKH &RPSUHKHQVLRQRI&RPSXWHU3URJUDPV SS� 1HZ<RUN1<86$ $&0KWWSGRLRUJ %URRNV5  7RZDUGVDWKHRU\RIWKHFRPSUHKHQVLRQRIFRPSXWHUSURJUDPV International Journal of Man-Machine Studies 18   � KWWSGRLRUJ6      %XVMDKQ7%HGQDULN5%HJHO$&URVE\03DWHUVRQ-6FKXOWH&�7DPP6   (\H 0RYHPHQWV LQ &RGH 5HDGLQJ 5HOD[LQJ WKH /LQHDU 2UGHU ,Q International Conference on Program Comprehension SWRDSSHDU  %XVMDKQ7 6FKXOWH& D 7KH8VHRI&RGH5HDGLQJLQ7HDFKLQJ3URJUDPPLQJ SS� 1HZ<RUN1<86$$&0KWWSGRLRUJ %XVMDKQ7 6FKXOWH& E 7KH8VHRI&RGH5HDGLQJLQ7HDFKLQJ3URJUDPPLQJ SS� 1HZ<RUN1<86$$&0KWWSGRLRUJ %XVMDKQ76FKXOWH& %XVMDKQ$  $QDO\VLVRIFRGHUHDGLQJWRJDLQPRUH LQVLJKW LQ SURJUDP FRPSUHKHQVLRQ ,Q Proceedings of the 11th Koli Calling International Conference on Computing Education Research - Koli Calling ’11 SS �  1HZ <RUN 1HZ <RUN 86$ $&0 3UHVV KWWSGRLRUJ %XVMDKQ76FKXOWH&6KDULI%6LPRQ%HJHO$+DQVHQ0�$QWURSRYD0  (\H7UDFNLQJLQ&RPSXWLQJ(GXFDWLRQ SS� 1HZ<RUN1<86$ $&0KWWSGRLRUJ &ULVWLQR)0DWK{W67KHHXZHV- *LOFKULVW,'  6FDQ0DWFK$QRYHO PHWKRGIRUFRPSDULQJIL[DWLRQVHTXHQFHVBehavior Research Methods 42   �KWWSGRLRUJ%50 &URVE\ 0   Natural Versus Computer Languages: A Reading Comparison 8QLYHUVLW\RI+DZDLL    &URVE\ 0 ( 6FKROW] -  :LHGHQEHFN6  7KH 5ROHV%HDFRQV 3OD\ LQ &RPSUHKHQVLRQIRU1RYLFHDQG([SHUW3URJUDPPHUV,Q14th Workshop of the Psychology of Programming Interest Group SS� &LWHVHHU )DQ4  The effects of beacons, comments, and tasks on program comprehension process in software maintenance8QLYHUVLW\RI0DU\ODQGDW%DOWLPRUH&RXQW\ &DWRQVYLOOH^0'`^86$` )ULW]7%HJHO$0�OOHU6&<LJLW(OOLRWW6 =�JHU0  8VLQJ3V\FKR SK\VLRORJLFDO0HDVXUHVWR$VVHVV7DVN'LIILFXOW\LQ6RIWZDUH'HYHORSPHQW,Q Proceedings of the 36th International Conference on Software Engineering SS � 1HZ<RUN1<86$$&0 *HOOHQEHFN(0 &RRN&5  An Investigation of Procedure and Variable Names As Beacons During Program Comprehension&RUYDOOLV2586$2UHJRQ 6WDWH8QLYHUVLW\ +DQVHQ0  (\H&RGH5HWULHYHG$SULOIURP +DQVHQ0(*ROGVWRQH5/ /XPVGDLQH$  :KDW0DNHV&RGH+DUGWR 8QGHUVWDQG"CoRRabs/1304.52575HWULHYHGIURPKWWSDU[LYRUJDEV -HWW\*+ 0D\  An Empirical Study Assessing the Impact of SeeIT3D on Comprehension<RXQJVWRZQ6WDWH8QLYHUVLW\<RXQJVWRZQ .LWFKHQKDP % $ '\ED 7  -RUJHQVHQ 0   (YLGHQFH%DVHG 6RIWZDUH (QJLQHHULQJ,QProceedings of the 26th International Conference on Software Engineering SS� :DVKLQJWRQ'&86$,(((&RPSXWHU6RFLHW\ 5HWULHYHGIURPKWWSGODFPRUJFLWDWLRQFIP"LG     /LEOLW%%HJHO$ 6ZHHWVHU(  &RJQLWLYH3HUVSHFWLYHVRQWKH5ROHRI1DPLQJ LQ &RPSXWHU 3URJUDPV ,Q Proceedings of the 18th Annual Psychology of Programming Workshop%ULJKWRQ(QJODQG8QLWHG.LQJGRP /LHQW]%36ZDQVRQ(% 7RPSNLQV*(  &KDUDFWHULVWLFVRIDSSOLFDWLRQ VRIWZDUH PDLQWHQDQFH Communications of the ACM 21   � KWWSGRLRUJ /LVWHU56HSSlOl26LPRQ%7KRPDV/$GDPV(6)LW]JHUDOG6�6DQGHUV .   $ PXOWLQDWLRQDO VWXG\ RI UHDGLQJ DQG WUDFLQJ VNLOOV LQ QRYLFH SURJUDPPHUV ACM SIGCSE Bulletin 36    KWWSGRLRUJ 3HQQLQJWRQ 1   6WLPXOXV 6WUXFWXUHV DQG 0HQWDO 5HSUHVHQWDWLRQV LQ ([SHUW &RPSUHKHQVLRQRI&RPSXWHU3URJUDPVCognitive Psychology  � 5D\QHU.&KDFH.+6ODWWHU\7- $VKE\-  (\H0RYHPHQWVDV 5HIOHFWLRQVRI&RPSUHKHQVLRQ3URFHVVHVLQ5HDGLQJScientific Studies of Reading 10  �KWWSGRLRUJV[VVUB 5RGHJKHUR30F0LOODQ&0F%XUQH\3:%RVFK1 '�0HOOR6   ,PSURYLQJDXWRPDWHGVRXUFHFRGHVXPPDUL]DWLRQYLDDQH\HWUDFNLQJVWXG\RI SURJUDPPHUV,QProceedings of the 36th International Conference on Software Engineering SS� $&0KWWSGRLRUJ 6FKXOWH & &OHDU 7 7DKHUNKDQL $ %XVMDKQ 7  3DWHUVRQ - +   $Q LQWURGXFWLRQWRSURJUDPFRPSUHKHQVLRQIRUFRPSXWHUVFLHQFHHGXFDWRUV S  $&03UHVVKWWSGRLRUJ    6KDULI%)DOFRQH0 0DOHWLF-,  $QH\HWUDFNLQJVWXG\RQWKHUROHRIVFDQ WLPHLQILQGLQJVRXUFHFRGHGHIHFWV,QProceedings of the Symposium on Eye Tracking Research and Applications - ETRA ’12 SS� 1HZ<RUN1HZ <RUN86$$&03UHVVKWWSGRLRUJ 6KDULI% .DJGL+  2QWKH8VHRI(\H7UDFNLQJLQ6RIWZDUH7UDFHDELOLW\ SS � 1HZ<RUN1<86$$&0KWWSGRLRUJ 6KDULI% 0DOHWLF-,  $QH\HWUDFNLQJVWXG\RQFDPHO&DVHDQGXQGHUBVFRUH LGHQWLILHUVW\OHV,QProceedings of the 2010 IEEE 18th International Conference on Program Comprehension SS �  ,((( KWWSGRLRUJ,&3& 6LHJPXQG-.lVWQHU&$SHO63DUQLQ&%HWKPDQQ$/HLFK7�%UHFKPDQQ $  8QGHUVWDQGLQJXQGHUVWDQGLQJVRXUFHFRGHZLWKIXQFWLRQDOPDJQHWLF UHVRQDQFHLPDJLQJProceedings of the 36th International Conference on Software Engineering�KWWSGRLRUJ 6RORZD\(/DPSHUW5/HWRYVN\6/LWWPDQ' 3LQWR-  'HVLJQLQJ 'RFXPHQWDWLRQWR&RPSHQVDWHIRU'HORFDOL]HG3ODQVCommun. ACM31   �KWWSGRLRUJ 7XUQHU5)DOFRQH06KDULI% /D]DU$  $QH\HWUDFNLQJVWXG\DVVHVVLQJ WKH FRPSUHKHQVLRQ RI F DQG 3\WKRQ VRXUFH FRGH ,Q Proceedings of the Symposium on Eye Tracking Research and Applications SS� 6DIHW\ +DUERU)ORULGD$&0KWWSGRLRUJ    8ZDQR+1DNDPXUD00RQGHQ$ 0DWVXPRWR.  $QDO\]LQJ,QGLYLGXDO 3HUIRUPDQFHRI6RXUFH&RGH5HYLHZ8VLQJ5HYLHZHUV�(\H0RYHPHQW SS�  1HZ<RUN1<86$$&0KWWSGRLRUJ :DOWHUV%)DOFRQH06KLEEOH$ 6KDULI%  7RZDUGVDQH\HWUDFNLQJ HQDEOHG ,'( IRU VRIWZDUH WUDFHDELOLW\ WDVNV ,Q Proceedings of the 2013 International Workshop on Traceability in Emerging Forms of Software Engineering SS� ,(((KWWSGRLRUJ7()6( :DOWHUV%6KDIIHU76KDULI% .DJGL+  &DSWXULQJVRIWZDUHWUDFHDELOLW\ OLQNV IURP GHYHORSHUV� H\H JD]HV ,Q Proceedings of the 22nd International Conference on Program Comprehension SS �  $&0 KWWSGRLRUJ