%PDF-1.3 %âãÏÓ 177 0 obj << /Linearized 1 /O 179 /H [ 2613 984 ] /L 497671 /E 275506 /N 12 /T 494012 >> endobj xref 177 100 0000000016 00000 n 0000002352 00000 n 0000003597 00000 n 0000004112 00000 n 0000004179 00000 n 0000004276 00000 n 0000004373 00000 n 0000004485 00000 n 0000004643 00000 n 0000004835 00000 n 0000004953 00000 n 0000005077 00000 n 0000005230 00000 n 0000005387 00000 n 0000005546 00000 n 0000005726 00000 n 0000005823 00000 n 0000005949 00000 n 0000006073 00000 n 0000006200 00000 n 0000006333 00000 n 0000006450 00000 n 0000006580 00000 n 0000006712 00000 n 0000006901 00000 n 0000007067 00000 n 0000007233 00000 n 0000007330 00000 n 0000007466 00000 n 0000007658 00000 n 0000007787 00000 n 0000007926 00000 n 0000008051 00000 n 0000008186 00000 n 0000008312 00000 n 0000008412 00000 n 0000008526 00000 n 0000008657 00000 n 0000008816 00000 n 0000008960 00000 n 0000009218 00000 n 0000009600 00000 n 0000010314 00000 n 0000010914 00000 n 0000019452 00000 n 0000020216 00000 n 0000020618 00000 n 0000020940 00000 n 0000023440 00000 n 0000029729 00000 n 0000030274 00000 n 0000030372 00000 n 0000030812 00000 n 0000030929 00000 n 0000030952 00000 n 0000032046 00000 n 0000032584 00000 n 0000032697 00000 n 0000039349 00000 n 0000039783 00000 n 0000039805 00000 n 0000040795 00000 n 0000041101 00000 n 0000041456 00000 n 0000045145 00000 n 0000045167 00000 n 0000046196 00000 n 0000046218 00000 n 0000047201 00000 n 0000047310 00000 n 0000047593 00000 n 0000047976 00000 n 0000048147 00000 n 0000048442 00000 n 0000049663 00000 n 0000049685 00000 n 0000050666 00000 n 0000052489 00000 n 0000052716 00000 n 0000052826 00000 n 0000053119 00000 n 0000054426 00000 n 0000054694 00000 n 0000055014 00000 n 0000055036 00000 n 0000055961 00000 n 0000056583 00000 n 0000056831 00000 n 0000057075 00000 n 0000057097 00000 n 0000058078 00000 n 0000058100 00000 n 0000059011 00000 n 0000274291 00000 n 0000274342 00000 n 0000274449 00000 n 0000274502 00000 n 0000274574 00000 n 0000002613 00000 n 0000003575 00000 n trailer << /Size 277 /Info 171 0 R /Root 178 0 R /Prev 494001 /ID[] >> startxref 0 %%EOF 178 0 obj << /FICL:Enfocus 173 0 R /PageMode /UseOutlines /Outlines 180 0 R /Dests 170 0 R /Metadata 176 0 R /Pages 172 0 R /OCProperties << /D << /RBGroups [ ] /Order [ ] >> /OCGs [ 271 0 R ] >> /OpenAction [ 179 0 R /Fit ] /Type /Catalog >> endobj 275 0 obj << /S 642 /T 951 /O 1007 /E 1023 /Filter /FlateDecode /Length 276 0 R >> stream H‰b```e`à2e`g`Ídf@ a6 (Ç6…ÍÍ< ·h®èoµjà–ÝrÄ6 ¨.»¥‚E¿AQ`å‰Ì>óûÓŒ§1'•jI›Üm}½Ôg¡]ÎËŠ‚’ÝEWw|Üù8hî4UÉ¢¢„£Uoþ‡,Ýu5[Ú"»ìêéÂÅc{sm¿ì·Ð[ï–eêÉõáq¯5üïoÎ9šî=ÕìêíiÝgÚ5³¾^j°Ô6jʍ5a[7·½ Uõ™”³óÚº;½yv1sv¿¾zÕiÊ›ébŽF]m‘¶dU±†ÿî¬'µg¬K<­µ•+R'jn}mÕ¦Ò™Z-¿¦¼º»µÈvÇ)‘å©Ó§¹éˆëÏÛÖr´hšŠ×̐›—ÞÝÚ)_’ʲln]\¤„alXÍ*aÕU>µ¶®V;w¸yéêíÊyýFµ,S½RzU§«®›Î{k]ÒÞ¦£Ù¹±;¥î”¾×L:iÕ½mûv©`Ö×SVøì9%|Û4ïÑÛÕׇ™ ’‹‹KhhhXZGƒ  1˜¸t€xÌÆ@I×д´´ŒŽŽ¨ µ40€(‚ŠD˜ÈJÆ *))©mEH+Üáâ‚lÐu‚‚ÂÆP5p¥JJ@&ÞTd—3°}QÒ @\Ö&Æ ÄPíÀtQ‡Á›A¬£iPV²Ã€I†¡“A½eã†4)¶Œ59Ò \L„°+ g3È:°0aÈ`n`=À¸¨K®ï ë¦-šD¸g õy3Ìzø†‰˜Ö¥€|FFÆÉgé@Wð+0Ê0;<5 î{ ßF€1†‘•!ʁ3€Å…e=PÄúƒŒ ÃY s>ˆˆ0|cðŸSƒEï ®ý@5æs•3dD?0œ`ÜÀ ÁÀ›ÀæÂØÌË þµ‡QœA•Ç€©„á>ƒžoæ?†Öãk`ÐÈ?`taðÇÌ@ýìz•@šøó8Xf9»ÇyP‚ ô«³J\²¾ùmqº(Û è –jPÌ00ÉéUìÞq #4² ‹4'Å endstream endobj 276 0 obj 852 endobj 179 0 obj << /CropBox [ 0 0 612 792 ] /Annots [ 214 0 R ] /Parent 172 0 R /Contents [ 231 0 R 237 0 R 242 0 R 244 0 R 252 0 R 261 0 R 266 0 R 268 0 R ] /Rotate 0 /MediaBox [ 0 0 612 792 ] /Resources << /ExtGState << /R16 270 0 R /R1 273 0 R >> /Font << /T1_8 254 0 R /T1_3 222 0 R /T1_7 259 0 R /T1_0 226 0 R /T1_2 217 0 R /T1_1 218 0 R /T1_5 239 0 R /T1_4 232 0 R /TT0 250 0 R /T1_6 248 0 R /T1_9 264 0 R >> /XObject << /Fm8 274 0 R >> /ProcSet [ /PDF /Text ] >> /Type /Page /Thumb 143 0 R >> endobj 180 0 obj << /Count 10 /First 181 0 R /Last 182 0 R >> endobj 181 0 obj << /Title (Introduction) /Dest /section.1 /Parent 180 0 R /Next 210 0 R >> endobj 182 0 obj << /Title (Conclusions) /Dest /section.10 /Parent 180 0 R /Prev 183 0 R >> endobj 183 0 obj << /Title (Related Work) /Dest /section.9 /Parent 180 0 R /Prev 184 0 R /Next 182 0 R >> endobj 184 0 obj << /Title (A Call to Action) /Dest /section.8 /Count -4 /Parent 180 0 R /Prev 185 0 R /Next 183 0 R /First 186 0 R /Last 187 0 R >> endobj 185 0 obj << /Title (Solutions: Detecting and Avoiding Measurement Bias) /Dest /section.7 /Count -3 /Parent 180 0 R /Prev 190 0 R /Next 184 0 R /First 191 0 R /Last 192 0 R >> endobj 186 0 obj << /Title (Diverse Evaluation Workloads) /Dest /subsection.8.1 /Parent 184 0 R /Next 189 0 R >> endobj 187 0 obj << /Title (More cooperation from the hardware) /Dest /subsection.8.4 /Parent 184 0 R /Prev 188 0 R >> endobj 188 0 obj << /Title (More information from the hardware manufacturers) /Dest /subsection.8.3 /Parent 184 0 R /Prev 189 0 R /Next 187 0 R >> endobj 189 0 obj << /Title (Identify more ways of randomizing experimental setup) /Dest /subsection.8.2 /Parent 184 0 R /Prev 186 0 R /Next 188 0 R >> endobj 190 0 obj << /Title (Literature Review) /Dest /section.6 /Count -3 /Parent 180 0 R /Prev 196 0 R /Next 185 0 R /First 197 0 R /Last 198 0 R >> endobj 191 0 obj << /Title (Evaluate innovations in many experimental setups) /Dest /subsection.7.1 /Count -2 /Parent 185 0 R /Next 193 0 R /First 194 0 R /Last 195 0 R >> endobj 192 0 obj << /Title (Summary) /Dest /subsection.7.3 /Parent 185 0 R /Prev 193 0 R >> endobj 193 0 obj << /Title (Using causal analysis) /Dest /subsection.7.2 /Parent 185 0 R /Prev 191 0 R /Next 192 0 R >> endobj 194 0 obj << /Title (Using a large benchmark suite) /Dest /subsubsection.7.1.1 /Parent 191 0 R /Next 195 0 R >> endobj 195 0 obj << /Title (Experimental setup randomization) /Dest /subsubsection.7.1.2 /Parent 191 0 R /Prev 194 0 R >> endobj 196 0 obj << /Title (Measurement Bias is Unpredictable) /Dest /section.5 /Parent 180 0 R /Prev 200 0 R /Next 190 0 R >> endobj 197 0 obj << /Title (Papers that use simulations) /Dest /subsection.6.1 /Parent 190 0 R /Next 199 0 R >> endobj 198 0 obj << /Title (Papers that acknowledge measurement bias) /Dest /subsection.6.3 /Parent 190 0 R /Prev 199 0 R >> endobj 199 0 obj << /Title (Papers that report speedups) /Dest /subsection.6.2 /Parent 190 0 R /Prev 197 0 R /Next 198 0 R >> endobj 200 0 obj << /Title (Measurement Bias is Significant and Commonplace) /Dest /section.4 /Count -4 /Parent 180 0 R /Prev 201 0 R /Next 196 0 R /First 202 0 R /Last 203 0 R >> endobj 201 0 obj << /Title (Experimental Methodology) /Dest /section.3 /Count -3 /Parent 180 0 R /Prev 210 0 R /Next 200 0 R /First 211 0 R /Last 212 0 R >> endobj 202 0 obj << /Title (Measurement bias due to link order) /Dest /subsection.4.1 /Count -2 /Parent 200 0 R /Next 205 0 R /First 208 0 R /Last 209 0 R >> endobj 203 0 obj << /Title (Summary) /Dest /subsection.4.4 /Parent 200 0 R /Prev 204 0 R >> endobj 204 0 obj << /Title (Did gcc cause measurement bias?) /Dest /subsection.4.3 /Parent 200 0 R /Prev 205 0 R /Next 203 0 R >> endobj 205 0 obj << /Title (Measurement bias due to UNIX environment size) /Dest /subsection.4.2 /Count -2 /Parent 200 0 R /Prev 202 0 R /Next 204 0 R /First 206 0 R /Last 207 0 R >> endobj 206 0 obj << /Title (The extent of the measurement bias) /Dest /subsubsection.4.2.1 /Parent 205 0 R /Next 207 0 R >> endobj 207 0 obj << /Title (The potential causes of the measurement bias) /Dest /subsubsection.4.2.2 /Parent 205 0 R /Prev 206 0 R >> endobj 208 0 obj << /Title (The extent of measurement bias) /Dest /subsubsection.4.1.1 /Parent 202 0 R /Next 209 0 R >> endobj 209 0 obj << /Title (The potential causes of measurement bias) /Dest /subsubsection.4.1.2 /Parent 202 0 R /Prev 208 0 R >> endobj 210 0 obj << /Title (Origin of Measurement Bias) /Dest /section.2 /Parent 180 0 R /Prev 181 0 R /Next 201 0 R >> endobj 211 0 obj << /Title (Benchmarks) /Dest /subsection.3.1 /Parent 201 0 R /Next 213 0 R >> endobj 212 0 obj << /Title (Following best practices) /Dest /subsection.3.3 /Parent 201 0 R /Prev 213 0 R >> endobj 213 0 obj << /Title (Measurement infrastructure) /Dest /subsection.3.2 /Parent 201 0 R /Prev 211 0 R /Next 212 0 R >> endobj 214 0 obj << /Type /Annot /C [ 0 1 0 ] /H /I /Border [ 0 0 0.72 ] /Rect [ 368.90401 104.664 375.936 113.436 ] /Dest /cite.m5 /Subtype /Link >> endobj 215 0 obj << /Type /Encoding /BaseEncoding /WinAnsiEncoding /Differences [ 2 /multiply 6 /plusminus 102 /braceleft /braceright ] >> endobj 216 0 obj << /Type /FontDescriptor /FontName /KRFLDL+CMSY10 /FontBBox [ 0 -250 721 750 ] /Flags 4 /Ascent 750 /CapHeight 750 /Descent -250 /ItalicAngle 0 /StemV 108 /CharSet (/braceleft/braceright/multiply/plusminus) /FontFile3 221 0 R >> endobj 217 0 obj << /BaseFont /KRFLDL+CMSY10 /FontDescriptor 216 0 R /Type /Font /FirstChar 2 /LastChar 103 /Widths [ 778 0 0 0 778 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 500 500 ] /Encoding 215 0 R /Subtype /Type1 >> endobj 218 0 obj << /BaseFont /YTGZHD+NimbusRomNo9L-Regu /FontDescriptor 219 0 R /Type /Font /FirstChar 2 /LastChar 180 /Widths [ 556 556 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 333 0 0 500 833 0 333 333 333 0 564 250 333 250 278 500 500 500 500 500 500 500 500 500 500 278 278 0 0 0 444 0 722 667 667 722 611 556 722 722 333 389 722 611 889 722 722 556 0 667 556 611 722 722 944 722 722 611 333 0 333 0 0 0 444 500 444 500 444 333 500 500 278 278 500 278 778 500 500 500 500 333 389 278 500 500 722 500 500 444 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 444 444 0 500 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 333 ] /Encoding 229 0 R /Subtype /Type1 >> endobj 219 0 obj << /Type /FontDescriptor /FontName /YTGZHD+NimbusRomNo9L-Regu /FontBBox [ -70 -218 1000 727 ] /Flags 4 /Ascent 727 /CapHeight 727 /Descent -218 /ItalicAngle 0 /StemV 150 /MissingWidth 250 /CharSet (/A/B/C/D/E/F/G/H/I/J/K/L/M/N/O/P/R/S/T/U/V/W/X/Y/Z/a/acute/b/bracketleft\ /bracketright/c/colon/comma/d/dollar/e/eight/emdash/endash/exclam/f/fi/f\ ive/fl/four/g/h/hyphen/i/j/k/l/m/n/nine/o/one/p/parenleft/parenright/per\ cent/period/plus/q/question/quotedblleft/quotedblright/quoteright/r/s/se\ micolon/seven/six/slash/t/three/two/u/v/w/x/y/z/zero) /FontFile3 220 0 R >> endobj 220 0 obj << /Filter /FlateDecode /Subtype /Type1C /Length 8444 >> stream xœyiXS×ÚöŽ1{ïÖ¡-i*›¤£¶Ö±ZQëŒ3*"(à€ˆa† $$„0-Æ0d æyEQœ‡V«u¨CµóàikÛÓµé¢ßy׆¶ç\ïéŸ÷». kÚëyîç¾ïgÁ!&N 8{xô¡x…·,z»l©çoih<ûë×73cóo%û„xExLè¿OC' Š;%UŒíËFåß÷“üÇ­ƒäò(õØZÙج¿ÎW*¤Q!sÇ+Á™ Š‘àäJ<%lv£‚âþ{äßùúÿË A>žkÔ1Á~ÛÖ&Éûo_'—ìðˆ ñZºAæ½Q¾kS|„Ïæ„Hß-‰QA»·ª¢íY>ÿ•Þ–…3‹ÞœµjÑk‹‹÷¾5{Â’7^àîwŸSºtî²y/ÄâE"€ØAxK‰¹ÄKÄ^‹XO,#æ/;‰ Ärb>ñ áMl$¯»ˆMÄL‡ØL¼IÌ"|‰-Ä"â5b7±•XL¼Nì!<‰·ˆÙ„±XK,!Þ ü‰íÄ:˜AAáijĄ„OωÄÛÄ4b2M¸Sˆ D±’S 1xŠp#ž&x„/®b"Þð(g§uÂÄ Ü'¹íÜÇWLâÍç9Ii!¥VSCô[tû“ž0>9õÉèI“º'ož<