@inproceedings{b11a2bc922724fdfbc699bf5709b7127,
title = "Beyond the medium: Rethinking information literacy through crowdsourced analysis",
abstract = "Information literacy encompasses a range of information evaluation skills for the purpose of making judgments. In the context of crowdsourcing, divergent evaluation criteria might introduce bias into collective judgments. Recent experiments have shown that crowd estimates can be swayed by social influence. This might be an unanticipated effect of media literacy training: encouraging readers to critically evaluate information falls short when their judgment criteria are unclear and vary among social groups. In this exploratory study, we investigate the criteria used by crowd workers in reasoning through a task. We crowdsourced evaluation of a variety of information sources, identifying multiple factors that may affect individual's judgment, as well as the accuracy of aggregated crowd estimates. Using a multi-method approach, we identified relationships between individual information assessment practices and analytical outcomes in crowds, and propose two analytic criteria, relevance and credibility, to optimize collective judgment in complex analytical tasks.",
author = "Olga Boichak and Jordan Canzonetta and Niraj Sitaula and Brian McKernan and Sarah Taylor and Patr{\'i}cia Rossini and Clegg, {Benjamin A.} and Kate Kenski and Martey, {Rosa Mikeal} and Nancy McCracken and Carsten {\O}sterlund and Roc Myers and James Folkestad and Jennifer Stromer-Galley",
note = "Funding Information: This research was supported by the Intelligence Advanced Research Projects Activity (IARPA) via the Department of Interior National Business Center contract number 2017-16121900004. The U.S. Government is authorized to reproduce and distribute reprints for Government purposes notwithstanding any copyright annotation thereon. Disclaimer: The views and conclusions expressed herein are those of the authors and should not be interpreted as necessarily representing the official policies or endorsements, either expressed or implied, of IARPA, DoI/NBC, or the U.S. Government. The authors thank SRC Inc. and its director Debi Plochocki for collaboration on creating TRACE, the analysis software used to collect data for this study. Funding Information: This research was supported by the Intelligence Advanced Research Projects Activity (IARPA) via the Department of Interior National Business Center contract number 2017-16121900004. The U.S. Government is authorized to reproduce and distribute reprints for Government purposes notwithstanding any copyright annotation thereon. Publisher Copyright: {\textcopyright} 2019 IEEE Computer Society. All rights reserved.; 52nd Annual Hawaii International Conference on System Sciences, HICSS 2019 ; Conference date: 08-01-2019 Through 11-01-2019",
year = "2019",
language = "English (US)",
series = "Proceedings of the Annual Hawaii International Conference on System Sciences",
publisher = "IEEE Computer Society",
pages = "420--429",
editor = "Bui, {Tung X.}",
booktitle = "Proceedings of the 52nd Annual Hawaii International Conference on System Sciences, HICSS 2019",
address = "United States",
}