@inproceedings{gupta-iconf25,
title = {{Finding Pareto Trade-offs in Fair and Accurate Detection of Toxic Speech}},
author = {Gupta, Soumyajit and Kovatchev, Venelin and Das, Anubrata and De-Arteaga, Maria and Lease, Matthew},
booktitle = {{Proceedings of the iConference: the 20th Annual Meeting of the International iSchools Organization}},
year = {2025},
techreport = {https://arxiv.org/abs/2204.07661}
}
@techreport{liu-arxiv24,
title = {{Exploring Multidimensional Checkworthiness: Designing AI-assisted Claim Prioritization for Human Fact-checkers}},
author = {Houjiang Liu and Jacek Gwizdka and Matthew Lease},
institution = {The University of Texas at Austin},
month = {December 11},
year = {2024},
techreport = {https://arxiv.org/abs/2412.08185},
number = {\href{https://arxiv.org/abs/2412.08185}{arXiv:2412.08185}}
}
@techreport{shi-arxiv24,
title = {{Argumentative Experience: Reducing Confirmation Bias on Controversial Issues through LLM-Generated Multi-Persona Debates}},
author = {Li Shi and Houjiang Liu and Yian Wong and Utkarsh Mujumdar and Dan Zhang and Jacek Gwizdka and Matthew Lease},
institution = {The University of Texas at Austin},
month = {December 5},
year = {2024},
techreport = {https://arxiv.org/abs/2412.04629},
number = {\href{https://arxiv.org/abs/2412.04629}{arXiv:2412.04629}}
}
@article{Neumann2024-js,
title = {Informational Justice in {AI}-Assisted Fact-Checking},
author = {Neumann, Terrence and De-Arteaga, Maria and Fazelpour, Sina and
Lease, Matthew and Saar-Tsechansky, Maytal},
journal = {Social Science Research Network (SSRN)},
month = october,
year = 2024,
techreport = {https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4996153}
}
@inproceedings{kambhatla-emnlp24,
title = {{Promoting Constructive Deliberation: Reframing for Receptiveness}},
author = {Gauri Kambhatla and Matthew Lease and Ashwin Rajadesingan},
booktitle = {Findings of the Association for Computational Linguistics: EMNLP},
year = {2024},
url = {https://aclanthology.org/2024.findings-emnlp.294/},
pages = {5110-5132},
techreport = {https://arxiv.org/abs/2405.15067}
}
@inproceedings{su-bbnlp24,
title = {{Wrapper Boxes: Faithful Attribution of Model Predictions to Training Data}},
author = {Yiheng Su and Junyi Jessy Li and Matthew Lease},
booktitle = {Proceedings of the 7th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for Natural Language Processing},
year = {2024},
confurl = {https://blackboxnlp.github.io/},
techreport = {https://arxiv.org/abs/2311.08644}
}
@inproceedings{subbu-sui24,
title = {{Annorama: Enabling Immersive At-Desk Annotation Experiences in Virtual Reality with 3D Point Cloud Dioramas}},
author = {Subramanian Chidambaram and Alex C Williams and Min Bai and Satyugjit Virk and Patrick Haffner and Matthew Lease and Li Erran Li},
booktitle = {Proceedings of the ACM Symposium on Spatial User Interaction (SUI)},
year = {2024},
url = {https://dl.acm.org/doi/10.1145/3677386.3682081},
techreport = {https://www.amazon.science/publications/annorama-enabling-immersive-at-desk-annotation-experiences-in-virtual-reality-with-3d-point-cloud-dioramas},
note = {13 pages}
}
@inproceedings{neumann-cist24,
title = {{Diverse, but Divisive: LLMs Can Exaggerate Gender Differences in Opinion Related to Harms of Misinformation}},
author = {Terrence Neumann and Maria De-Arteaga and Sooyong Lee and Matthew Lease and Sina Fazelour},
booktitle = {\href{https://sites.google.com/view/cist2024/}{Conference on Information Systems and Technology (CIST)}},
year = {2024},
techreport = {https://arxiv.org/abs/2401.16558}
}
@techreport{gupta-arxiv24,
title = {{Fairly Accurate: Optimizing Accuracy Parity in Fair Target-Group Detection}},
author = {Soumyajit Gupta and Venelin Kovatchev and Maria De-Arteaga and Matthew Lease},
institution = {The University of Texas at Austin},
month = {July 16},
year = {2024},
techreport = {https://arxiv.org/abs/2407.11933},
number = {arXiv:2407.11933}
}
@mastersthesis{mujumdar-thesis24,
author = {Utkarsh Mujumdar},
title = {Designing a Multi-Perspective Search System Using Large Language Models and Retrieval Augmented Generation},
school = {School of Information, The University of Texas at Austin},
year = {2024},
news = {https://www.ischool.utexas.edu/news/2024-capstone-award-winners},
url = {../papers/mujumdar-thesis24.pdf},
note = {\bf Winner: Dean's Choice Award.}
}
@mastersthesis{wong-thesis24,
author = {Yian Wong},
title = {Exploring Multiple Perspectives to Mitigate Cognitive Biases through an Integrated Interface to Language Models},
school = {Dept.\ of Computer Science, The University of Texas at Austin},
year = {2024},
url = {../papers/wong-thesis24.pdf}
}
@article{liu-cscw24-pacm,
author = {Liu, Houjiang and Das, Anubrata and Boltz, Alexander and Zhou, Didi and Pinaroc, Daisy and Lease, Matthew and Lee, Min Kyung},
title = {{Human-centered NLP Fact-checking: Co-Designing with Fact-checkers using Matchmaking for AI}},
year = {2024},
volume = {8\,},
number = {CSCW2},
journal = {Proceedings of the ACM on Human-Computer Interaction},
articleno = {423},
pages = {1-44},
doi = {10.1145/3686962},
url = {https://mattlease.com/papers/liu-cscw24.pdf},
note = {\faAward\ {\bf Best Paper Honorable Mention}}
}
@inproceedings{kovatchev-naacl24,
title = {{Benchmark Transparency: Measuring the Impact of Data on Evaluation}},
author = {Venelin Kovatchev and Matthew Lease},
booktitle = {Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT)},
year = {2024},
url = {https://aclanthology.org/2024.naacl-long.86/},
source = {https://github.com/venelink/benchmark_transparency},
techreport = {https://arxiv.org/abs/2404.00748},
pages = {1536-1551}
}
@inproceedings{williams-iui24,
title = {{Snapper: Accelerating Bounding Box Annotation in Object Detection Tasks with Find-and-Snap Tooling}},
author = {Alex C Williams and Min Bai and Jonathan Buck and Tristan J McKinney and Amy Rechkemmer and
Koushik Kalyanaraman and Matthew Lease and Patrick Haffner and
Xiong Zhou and Kumar Chellapilla and Li Erran Li},
booktitle = {The 29th ACM Intl.\ Conference on Intelligent User Interfaces (IUI)},
year = {2024},
pages = {471-488},
year = {2024},
isbn = {9798400705083},
doi = {10.1145/3640543.3645162},
url = {https://www.amazon.science/publications/snapper-accelerating-bounding-box-annotation-in-object-detection-tasks-with-find-and-snap-tooling},
slides = {https://utexas.box.com/v/snapper-iui24-slides}
}
@mastersthesis{thapa-thesis23,
author = {Sukanya Thapa},
title = {Enhancing Worker Management and Supporting External Tasks in Crowdsourced Data Labeling},
school = {School of Information, University of Texas at Austin},
year = {2023},
url = {https://repositories.lib.utexas.edu/items/57046971-b0b5-42f3-a84b-b1f368d095f1}
}
@article{oala-dmlr24,
title = {{DMLR: Data-centric Machine Learning Research -- Past, Present and Future}},
author = {Luis Oala and Manil Maskey and Lilith Bat-Leah and Alicia Parrish and Nezihe Merve G{\"u}rel and Tzu-Sheng Kuo and Yang Liu and Rotem Dror and Danilo Brajovic and Xiaozhe Yao and Max Bartolo and William A Gaviria Rojas and Ryan Hileman and Rainier Aliment and Michael W. Mahoney and Meg Risdal and Matthew Lease and Wojciech Samek and Debojyoti Dutta and Curtis G Northcutt and Cody Coleman and Braden Hancock and Bernard Koch and Girmaw Abebe Tadesse and Bojan Karla{\v{s}} and Ahmed Alaa and Adji Bousso Dieng and Natasha Noy and Vijay Janapa Reddi and James Zou and Praveen Paritosh and Mihaela van der Schaar and Kurt Bollacker and Lora Aroyo and Ce Zhang and Joaquin Vanschoren and Isabelle Guyon and Peter Mattson},
journal = {{Journal of Data-centric Machine Learning Research (DMLR)}},
year = {2024},
journal = {\href{https://data.mlr.press/}{Journal of Data-centric Machine Learning Research} (DMLR)},
confurl = {https://data.mlr.press/volumes/01.html},
volume = {1},
url = {https://data.mlr.press/assets/pdf/v01-5.pdf},
techreport = {https://arxiv.org/abs/2311.13028}
}
@article{braylan-jair23,
author = {Alexander Braylan and Madalyn Marabella and Omar Alonso and Matthew Lease},
title = {{A General Model for Aggregating Annotations Across Simple, Complex, and Multi-Object Annotation Tasks}},
journal = {Journal of Artificial Intelligence Research (JAIR)},
year = {2023},
month = {December},
volume = {78},
pages = {901-973},
source = {https://github.com/Praznat/annotationmodeling},
data = {https://github.com/Praznat/annotationmodeling},
url = {https://www.jair.org/index.php/jair/article/view/14388},
note = {{Presented at the 2024 Annual AAAI Conference on Artificial Intelligence.}}
}
@conference{danula-hcomp23,
author = {Danula Hettiachchi and Indigo Holcombe-James and Stephanie Livingstone and Anjalee de Silva and Matthew Lease and Flora D. Salim and Mark Sanderson},
title = {{How Crowd Worker Factors Influence Subjective Annotations: A Study of Tagging Misogynistic Hate Speech in Tweets}},
booktitle = {{Proceedings of the 11th AAAI Conference on Human Computation and Crowdsourcing (HCOMP)}},
pages = {38-50},
year = {2023},
url = {../papers/hettiachchi-hcomp23.pdf},
techreport = {https://arxiv.org/abs/2309.01288}
}
@conference{amy-hcomp23,
author = {Amy Rechkemmer and Alex C.\ Williams and Matthew Lease and Li Erran Li},
title = {{Characterizing Time Spent in Video Object Tracking Annotation Tasks: A Study of Task Complexity in Vehicle Tracking}},
booktitle = {{Proceedings of the 11th AAAI Conference on Human Computation and Crowdsourcing (HCOMP)}},
pages = {140-151},
year = {2023},
url = {https://www.amazon.science/publications/characterizing-time-spent-in-video-object-tracking-annotation-tasks-a-study-of-task-complexity-in-vehicle-tracking},
slides = {https://rechkamy.github.io/papers/HCOMP23_slides.pdf}
}
@mastersthesis{Su-thesis23,
author = {Yiheng Su},
title = {Wrapper Boxes for Increasing Model Interpretability via Example-based Explanations},
school = {Department of Computer Science, University of Texas at Austin},
year = {2023},
url = {https://repositories.lib.utexas.edu/handle/2152/121132}
}
@mastersthesis{Lee-thesis23,
author = {Sooyong Lee},
title = {Multi-Task Learning for Hate Speech Detection },
school = {Department of Computer Science, University of Texas at Austin},
year = {2023},
url = {https://repositories.lib.utexas.edu/handle/2152/119144}
}
@incollection{jie23-frontiers,
title = {{Editorial: ``Human-Centered AI: Crowd Computing''}},
author = {Jie Yang and Alessandro Bozzon and Ujwal Gadiraju and Matthew Lease},
year = {2023},
booktitle = {Frontiers in Artificial Intelligence},
series = {{\em Special Topic on Human-Centered AI: Crowd Computing}},
publisher = {Frontiers},
note = {DOI: 10.3389/frai.2023.1161006},
url = {https://www.frontiersin.org/articles/10.3389/frai.2023.1161006/full}
}
@incollection{keswani-hhai23,
title = {{Designing Closed-Loop Models for Task Allocation}},
booktitle = {2nd International Conference on Hybrid Human-Artificial Intelligence (HHAI)},
author = {Vijay Keswani and Elisa Celis and Krishnaram Kenthapadi and Matthew Lease},
pages = {17--32},
year = {2023},
publisher = {IOS Press},
url = {https://utexas.box.com/v/keswani-hhai23},
techreport = {https://arxiv.org/abs/2202.04718},
confurl = {https://hhai-conference.org/2023/},
slides = {https://utexas.box.com/v/keswani-hhai23-slides}
}
@techreport{gao2023learning,
title = {{Learning Complementary Policies for Human-AI Teams}},
author = {Gao, Ruijiang and Saar-Tsechansky, Maytal and De-Arteaga, Maria and Han, Ligong and Sun, Wei and Lee, Min Kyung and Lease, Matthew},
institution = {University of Texas at Austin},
year = {2023},
note = {February 6. arXiv:2302.02944},
url = {https://arxiv.org/abs/2302.02944}
}
@techreport{du2023voices,
title = {{Voices of Workers: Why a Worker-Centered Approach to Crowd Work Is Challenging}},
author = {Du, Caifan and Lease, Matthew},
institution = {University of Texas at Austin},
year = {2023},
note = {January 6. arXiv:2212.14471},
url = {https://arxiv.org/abs/2212.14471}
}
@inproceedings{gupta-webconf23,
title = {{{\em Same Same, But Different}: Conditional Multi-Task Learning for Demographic-Specific Toxicity Detection}},
author = {Soumyajit Gupta and Sooyong Lee and Maria De-Arteaga and Matthew Lease},
booktitle = {{Proceedings of the Web Conference}},
year = {2023},
pages = {3689-3700},
url = {https://dl.acm.org/doi/10.1145/3543507.3583290},
source = {https://github.com/smjtgupta/CondMTL},
techreport = {https://arxiv.org/abs/2302.07372},
video = {https://youtu.be/UPGC25JwxyM},
slides = {https://utexas.box.com/v/gupta-webconf23-slides},
note = {Additional, shorter \href{https://youtu.be/ajqUFXuDb5s}{video}}
}
@article{das-ipm23,
author = {Anubrata Das and Houjiang Liu and Venelin Kovatchev and Matthew Lease},
title = {{The State of Human-centered NLP Technology for Fact-checking}},
journal = {Information Processing \& Management},
volume = {60},
number = {2},
year = {2023},
issn = {0306-4573},
doi = {https://doi.org/10.1016/j.ipm.2022.103219},
url = {https://mattlease.com/papers/das-ipm23.pdf},
techreport = {https://arxiv.org/abs/2301.03056}
}
@inproceedings{turkmen-ecir23,
author = {T{\"u}rkmen, Mehmet Deniz and Matthew Lease and Mucahid Kutlu},
title = {{New Metrics to Encourage Innovation and Diversity in Information Retrieval Approaches}},
booktitle = {{Proceedings of the 45th European Conference on Information Retrieval (ECIR)}},
pages = {239--254},
year = {2023},
url = {https://utexas.box.com/shared/static/infb268yzdbu3opcubvycrhfry9fqsjv.pdf},
slides = {https://utexas.box.com/v/turkmen-ecir23-slides},
techreport = {https://arxiv.org/abs/2301.08062}
}
@inproceedings{gao-cist22,
title = {{Robust Human-AI Collaboration with Bandit Feedback}},
author = {Ruijiang Gao and Maytal Saar-Tsechansky and Maria De-Arteaga and Ligong Han and Min Kyung Lee and Wei Sun and Matthew Lease},
booktitle = {\href{https://sites.google.com/view/cist2022/}{Conference on Information Systems and Technology (CIST)}},
year = {2022},
note = {{\bf Best Student Paper \href{https://sites.google.com/view/cist2022/program}{Award}.} Further revised \& retitled as \url{https://arxiv.org/abs/2302.02944}.}
}
@techreport{turkmen22-tr,
title = {{A New Evaluation Metric Rewarding Information Retrieval of Hard Documents}},
author = {Mehmet Deniz T{\"u}rkmen and Matthew Lease and Mucahid Kutlu},
year = {2022},
institution = {University of Texas at Austin, Department of Computer Science},
number = {TR-22-01},
note = {January 14.},
url = {https://apps.cs.utexas.edu/apps/tech-reports/196693}
}
@inproceedings{das-ipm22,
title = {{The Need for Human-centered Design in Fact-checking Research}},
author = {Das, Anubrata and Houjiang Liu and Kovatchev, Venelin and Matthew Lease},
booktitle = {Proceedings of the 1st Information Processing \& Management (IP\&M) Conference},
year = {2022},
url = {https://utexas.box.com/shared/static/zo5tm5d7qzngsrdn0wb1l496isrusxo6.pdf},
confurl = {https://www.elsevier.com/events/conferences/information-processing-and-management-conference}
}
@inproceedings{lease-cr22,
author = {Matthew Lease},
title = {{A Better Way to Measure Annotator Agreement for Complex Tasks}},
booktitle = {{CloudResearch Conference on Innovations in Online Research}},
year = 2022,
confurl = {https://web.cvent.com/event/5513a3c0-4938-44c4-9382-e64bcf169edc/summary},
video = {https://www.youtube.com/watch?v=JHGbD13QTpo&t=940s},
slides = {https://utexas.box.com/v/lease-cloudresearch22-slides},
abstract = {Data annotation (aka labeling or coding) is fundamental and ubiquitous in both machine learning (ML) and behavioral sciences. With ML, annotated data enables training supervised learning models and evaluating accuracy. In behavioral science, content analysis codes participant responses for study. Across both, annotator agreement measures (AAMs) assess the extent of agreement between human annotators (e.g., professional labelers, crowdsourcing contributors, or researchers) in labeling data consistently. Establishing such consistency is often a precursor to any subsequent use of annotated data. One of the best known measures, Krippendorf's alpha (KA), usefully supports measurement across any number of annotators. In its most general form, it also supports any annotation task in which distance between annotations can be quantified. However, this form of KA cannot distinguish two distributions of annotation distances having the same mean, rendering it quite brittle in practice. My lab's contribution is three-fold. First, we show any annotation task's evaluation metric can be repurposed as a distance function to facilitate use of KA (achieving generality). Second, we propose a simple change in how distance distributions are compared (using the Kolmogorv-Smirnov test instead of comparing only the means) to boost measure sensitivity (improving robustness). Third, we perform the first benchmarking of KA's general form across a wide range of simulated and real annotation tasks, demonstrating its inaccuracies and our improvement to it. Our approach is trivial to implement, and we also provide opensource.},
note = {Presentation.}
}
@inproceedings{kovatchev-dadc22,
title = {Longhorns at DADC 2022: How many linguists does it take to fool a Question Answering model? A systematic approach to adversarial attacks},
author = {Kovatchev, Venelin and Chatterjee, Trina and Govindarajan, Venkata S and Chen, Jifan and Choi, Eunsol and Chronis, Gabriella and Das, Anubrata and Erk, Katrin and Lease, Matthew and Li, Junyi Jessy and others},
booktitle = {Proceedings of the First Workshop on Dynamic Adversarial Data Collection (DADC) at the Annual Conference of the North American Chapter of the Association for Computational Linguistics (NAACL)},
pages = {41--52},
year = {2022},
url = {https://aclanthology.org/2022.dadc-1.5/},
slides = {https://utexas.box.com/v/kovatchev-ddac22-slides},
poster = {https://utexas.box.com/v/kovatchev-ddac22-poster},
techreport = {https://arxiv.org/abs/2206.14729}
}
@mastersthesis{Zhou22,
author = {Didi Zhou},
title = {Leveraging Annotator Rationales for Active Learning with Transformers},
school = {Department of Computer Science, University of Texas at Austin},
year = {2022},
type = {Bachelor's Thesis},
url = {https://apps.cs.utexas.edu/apps/tech-reports/197132}
}
@phdthesis{rahman-dissertation21,
author = {Md Mustafizur Rahman},
title = {{Reliable and low-cost test collections construction using machine learning}},
school = {School of Information, University of Texas at Austin},
month = {August},
year = 2021,
url = {https://repositories.lib.utexas.edu/handle/2152/109164}
}
@inproceedings{gupta22-uai,
title = {{Learning a Neural Pareto Manifold Extractor with Constraints}},
author = {Soumyajit Gupta and Gurpreet Singh and Raghu Bollapragada and Matthew Lease},
booktitle = {Proceedings of the 38th International Conference on Uncertainty in Artificial Intelligence (UAI)},
year = {2022},
url = {https://utexas.app.box.com/v/gupta-uai2022},
techreport = {https://arxiv.org/abs/2110.15442},
poster = {https://utexas.box.com/v/gupta-uai22-poster},
video = {https://youtu.be/vCkuvVozGwM},
source = {https://github.com/smjtgupta/SUHNPF},
pages = {749--758}
}
@article{pradhan22-frontiers,
title = {In Search of Ambiguity: A Three-Stage Workflow Design to Clarify Annotation Guidelines for Crowd Workers},
author = {Vivek Krishna Pradhan and Mike Schaekermann and Matthew Lease},
year = {2022},
journal = {Frontiers in Artificial Intelligence},
techreport = {https://arxiv.org/abs/2112.02255},
url = {https://www.frontiersin.org/articles/10.3389/frai.2022.828187/full},
issn = {2624-8212},
note = {ISSN:2624-8212.}
}
@inproceedings{das-acl22,
author = {Anubrata Das and Chitrank Gupta and Venelin Kovatchev and Matthew Lease and Junyi Jessy Li},
title = {{{\sc ProtoTEx}: Explaining Model Decisions with Prototype Tensors}},
booktitle = {{Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (ACL)}},
year = {2022},
url = {https://aclanthology.org/2022.acl-long.213/},
source = {https://github.com/anubrata/ProtoTEx/},
pages = {2986--2997},
video = {https://youtu.be/QvPdYlsJGrg},
poster = {https://drive.google.com/file/d/10i69YGMfj2FxcPTu6NtUuJuhHsD3eIQC/view?usp=sharing},
slides = {https://utexas.box.com/v/das-acl22-slides},
techreport = {https://arxiv.org/abs/2204.05426}
}
@inproceedings{Shi-chiir22,
author = {Li Shi and Nilavra Bhattacharya and Anubrata Das and Matthew Lease and Jacek Gwizdka},
title = {{The Effects of Interactive AI Design on User Behavior: An Eye-tracking Study of Fact-checking COVID-19
Claims}},
booktitle = {Proceedings of the 7th ACM SIGIR Conference on Human Information,
Interaction and Retrieval (CHIIR)},
year = 2022,
pages = {315--320},
url = {https://utexas.box.com/v/shi-chiir2022},
techreport = {https://arxiv.org/abs/2202.08901},
poster = {https://utexas.box.com/v/shi-chiir22-poster},
video = {https://www.youtube.com/watch?v=aic030YwiLA},
source = {https://github.com/ixlab-ut/chiir-2022},
demo = {https://volt..utexas.edu/viz/fcweb_ui_demo.php}
}
@inproceedings{Braylan-web2022,
title = {{A General Measure for Annotator Agreement across Structured, Multi-object, and Free-text Annotation Tasks}},
author = {Alexander Braylan and Omar Alonso and Matthew Lease},
booktitle = {{Proceedings of the Web Conference}},
year = {2022},
url = {https://utexas.box.com/v/braylan-webconf22},
source = {https://github.com/Praznat/annotationmodeling},
data = {https://github.com/Praznat/annotationmodeling},
video = {https://youtu.be/g9sbJQSEW5M},
slides = {https://utexas.box.com/v/braylan-webconf22-slides},
pages = {1720--1730},
techreport = {https://arxiv.org/abs/2212.09503}
}
@book{hettiachchi21-workshop-proceedings,
editor = {Danula Hettiachchi and Mark Sanderson and Jorge Goncalves and Simo Hosio and Gabriella Kazai and Matthew Lease
and Mike Schaekermann and Emine Yilmaz},
title = {{Investigating and Mitigating Biases in Crowdsourced Data: Workshop Proceedings}},
publisher = {The 24th ACM Conference on Computer Supported Cooperative Work (CSCW)},
year = {2021},
confurl = {https://sites.google.com/view/biases-in-crowdsourced-data/papers},
url = {https://arxiv.org/abs/2111.14322},
note = {32 pages.}
}
@article{aroyo22,
title = {{Data Excellence for AI: Why Should You Care?}},
author = {Lora Aroyo and Matthew Lease and Praveen Paritosh and Mike Schaekermann},
journal = {ACM Interactions},
volume = {29},
number = {2},
year = {2022},
pages = {66--69},
note = {March-April},
techreport = {https://arxiv.org/abs/2111.10391},
url = {https://utexas.box.com/v/aroyo-ix2022}
}
@inproceedings{rahman-iconf22,
author = {Md Mustafizur Rahman and Mucahid Kutlu and Matthew Lease},
title = {{Understanding and Predicting Characteristics of Test Collections in Information Retrieval}},
booktitle = {{Proceedings of the 17th Annual iConference}},
isbn = {978-3-030-96960-8},
year = {2022},
pages = {136--148},
url = {https://utexas.box.com/v/rahman-iconf22},
confurl = {https://ischools.org/iConference},
techreport = {https://arxiv.org/pdf/2012.13292},
video = {https://www.youtube.com/watch?v=5e1vzi6AOy4},
source = {https://github.com/mdmustafizurrahman/Understanding-and-Predicting-the-Characteristics-of-Test-Collections-in-Information-Retrieval},
slides = {https://github.com/mdmustafizurrahman/mdmustafizurrahman.github.io/raw/master/paper/iConf_2022_Presentation_final.pptx}
}
@inproceedings{chaudhry-iconf22,
title = {{You Are What You Tweet: Profiling Users by Past Tweets to Improve Hate Speech Detection}},
author = {Prateek Chaudhry and Matthew Lease},
booktitle = {{Proceedings of the 17th Annual iConference}},
isbn = {978-3-030-96960-8},
year = {2022},
pages = {195--203},
url = {https://utexas.box.com/v/prateek-iconf22},
confurl = {https://ischools.org/iConference},
techreport = {http://arxiv.org/abs/arXiv:2012.09090},
slides = {https://utexas.box.com/v/prateek-iconf22-slides},
video = {https://youtu.be/kNP9BC3H0D4}
}
@inproceedings{braylan-bhcc21,
author = {Alexander Braylan and Matthew Lease},
title = {{Detecting Bias in Complex Annotations}},
booktitle = {Third symposium on Biases in Human Computation and Crowdsourcing (BHCC)},
year = 2021,
confurl = {https://www.bhcc-symposium.com/},
url = {https://utexas.box.com/v/braylan-bhcc21},
slides = {https://utexas.box.com/v/braylan-bhcc21-slides},
note = {Presentation.}
}
@inproceedings{lease-cscw21,
author = {Matthew Lease},
title = {{Designing Human-AI Partnerships for Annotation, Moderation, and Fact-Checking}},
booktitle = {Workshop on Human-Machine Partnerships in the Future of Work at the 24th ACM Conference on Computer Supported Cooperative Work (CSCW)},
year = 2021,
confurl = {https://sites.google.com/view/cscw2021workshop/},
url = {https://utexas.box.com/v/lease-cscw21-hmpfw}
}
@inproceedings{savaage-cscw21,
author = {Matthew Lease and Mohammad Hossein Jarrahi and Saiph Savage},
title = {{Designing for the Global Workers During the Pandemic}},
booktitle = {Workshop on The Global Labours of AI and Data Intensive Systems at the 24th ACM Conference on Computer Supported Cooperative Work (CSCW)},
year = 2021,
confurl = {https://globaltechandsociety.red/cscw2021/},
video = {https://utexas.box.com/v/savage-gladis21-video},
note = {Presentation.}
}
@inproceedings{das-knighttool21,
author = {Anubrata Das and Sooyong Lee and Nguyen, An Thanh and Aditya Kharosekar and Saumyaa Krishnan and Siddhesh
Krishnan and Elizabeth Tate and Byron C.\ Wallace and Matthew Lease},
title = {{ExFacto: An Explainable Fact-Checking Tool}},
booktitle = {{Knight Research Network Tool Demonstration Day}},
year = {2021},
slides = {https://docs.google.com/presentation/d/1cjGGAtEwjrf8KXWgwJtOqoJGd3WoXjcUGzC49YL28v4/edit?usp=sharing},
confurl = {https://www.cmu.edu/ideas-social-cybersecurity/events/krn-tool-demo.html},
video = {https://www.youtube.com/watch?v=1Ltdoctl8cE},
note = {Presentation.}
}
@inproceedings{lease-cr21,
author = {Matthew Lease},
title = {{Automated Models for Quantifying Centrality of Survey Responses}},
booktitle = {{CloudResearch Conference on Innovations in Online Research}},
year = 2021,
confurl = {https://web.cvent.com/event/d063e447-1f16-4f70-a375-5d6978b3feea/summary},
video = {https://youtu.be/OQPn32TNJ9o},
slides = {https://www.slideshare.net/mattlease/automated-models-for-quantifying-centrality-of-survey-responses},
abstract = {When collecting data online, an automated method to quantify relative centrality of participant responses can provide insights for quality assurance plus assessing responses and participants. For example, given a set of textual responses to a survey question, which responses are most normative vs. others? Which represent the greatest outliers? Over a set of questions, which participants provide the most normative or outlying responses overall? How might such automated quantitative assessment inform analysis of responses and participants? My lab has released an open source library (free for commercial use) that enables such centrality measures to be computed in a general way across arbitrary question types. We have already published several research articles on this work I will synthesize for the talk. For any question type (e.g., textual response), we quantify distance between two participant responses via a user-selected distance function. This can be a built-in distance function we provide (e.g., for text responses: Levenshtein, BLEU, or GLEU distance) or any arbitrary distance function provided by the user (e.g., computing semantic distance based on a BERT transformer). Using this specified distance function, we provide a suite of aggregation models of varying complexity (e.g., multi-dimensional scaling with Bayesian priors and hyperparameters) that score responses for each question and aggregate statistics across questions. We report findings across a range of question types, distance functions, and aggregation models.},
note = {Presentation.}
}
@inproceedings{rahman-dcai21,
author = {Md Mustafizur Rahman and Dinesh Balakrishnan and Dhiraj Murthy and Mucahid Kutlu and Matthew Lease},
title = {{Addressing Content Selection Bias in Creating Datasets for Hate Speech Detection}},
booktitle = {{Proceedings of the Workshop on Data-Centric AI at the 35th Conference on Neural Information Processing
Systems (NeurIPS)}},
year = {2021},
confurl = {https://eval.how/dcai2021/},
url = {https://utexas.box.com/v/rahman-dcai21},
video = {https://recorder-v3.slideslive.com/?share=53914&s=98a43f20-a0d5-4a48-b8ab-78a4ae4b5166},
slides = {https://utexas.box.com/v/rahman-dcai21-slides},
source = {https://github.com/mdmustafizurrahman/An-Information-Retrieval-Approach-to-Building-Datasets-for-Hate-Speech-Detection},
note = {4 pages.}
}
@inproceedings{keswani-bhcc21,
author = {Vijay Keswani and Matthew Lease and Krishnaram Kenthapadi},
title = {{Designing human-in-the-loop approaches for closed deferral pipelines}},
booktitle = {Third symposium on Biases in Human Computation and Crowdsourcing (BHCC)},
confurl = {https://www.bhcc-symposium.com/},
url = {https://utexas.box.com/v/keswani-bhcc21},
note = {10 pages.},
year = 2021
}
@techreport{singh-arxiv22,
title = {{The Case for Claim Difficulty Assessment in Automatic Fact Checking}},
author = {Prakhar Singh and Anubrata Das and Junyi Jessy Li and Matthew Lease},
year = {2022},
institution = {University of Texas at Austin},
month = {February 4},
url = {https://arxiv.org/pdf/2109.09689.pdf},
number = {arXiv:2109.09689}
}
@inproceedings{rahman21-neurips21,
author = {Md Mustafizur Rahman and Dinesh Balakrishnan and Dhiraj Murthy and Mucahid Kutlu and Matthew Lease},
title = {{An Information Retrieval Approach to Building Datasets for Hate Speech Detection}},
booktitle = {Proceedings of the Thirty-fifth Conference on Neural Information Processing Systems (NeurIPS): Datasets and
Benchmarks Track},
techreport = {https://arxiv.org/abs/2106.09775},
video = {https://recorder-v3.slideslive.com/?share=53926&s=3808f963-c756-4d20-8899-f568ff8b76a2},
slides = {https://utexas.box.com/v/rahman-neurips21-slides},
year = {2021},
url = {https://utexas.box.com/v/rahman-neurips21},
supp = {https://utexas.box.com/v/rahman-neurips21-supplementary},
note = {15 pages.},
poster = {https://utexas.box.com/v/rahman-neurips21-poster},
source = {https://github.com/mdmustafizurrahman/An-Information-Retrieval-Approach-to-Building-Datasets-for-Hate-Speech-Detection}
}
@inproceedings{fleischmann2013calling,
author = {Fleischmann, Kenneth R and Goggins, Sean P.\ and Howison, James and Lease, Matthew and Oard, Douglas W.},
title = {Calling all computer scientists and social scientists: Establishing a research agenda for computational social science},
booktitle = {{Proceedings of the iConference}},
year = {2013},
pages = {1035--1036},
url = {https://doi.org/10.9776/13338}
}
@book{ghosh16-hcomp,
editor = {Arpita Ghosh and Matthew Lease},
title = {Proceedings of the 4th AAAI Conference on Human Computation and Crowdsourcing (HCOMP)},
publisher = {Association for the Advancement of Artificial Intelligence (AAAI)},
year = {2016},
isbn = {978-1-57735-774-2},
confurl = {https://www.aaai.org/Library/HCOMP/hcomp16contents.php},
url = {https://www.aaai.org/Press/Proceedings/hcomp16.php},
note = {ISBN 978-1-57735-774-2, 290 pages.}
}
@book{bennett13-crowdconf,
editor = {Paul Bennett and Matthew Lease},
title = {Proceedings of the Research Track from the 4th Annual CrowdConf},
publisher = {Online},
year = {2013},
confurl = {http://ai.ischool.utexas.edu/crowdconf2013-research-track.mhtml},
url = {https://utexas.box.com/v/CrowdConf2013-Proceedings},
note = {21 pages.}
}
@inproceedings{gupta-ictir21,
title = {{Pareto Solutions vs Dataset Optima: Concepts and Methods for Optimizing Competing Objectives with Constraints in Retrieval}},
author = {Soumyajit Gupta and Gurpreet Singh and Anubrata Das and Matthew Lease},
booktitle = {{Proceedings of The 7th ACM SIGIR International Conference on the Theory of Information Retrieval (ICTIR)}},
year = {2021},
url = {https://utexas.box.com/v/gupta-ictir21},
poster = {https://utexas.box.com/v/gupta-ictir21-poster},
slides = {https://utexas.box.com/v/gupta-ictir21-slides},
confurl = {https://dl.acm.org/conference/ictir},
pages = {43--52},
video = {https://youtu.be/FRlInj2zoMk}
}
@inproceedings{braylan-kdd21,
author = {Braylan, Alexander and Lease, Matthew},
title = {Aggregating Complex Annotations via Merging and Matching},
year = {2021},
booktitle = {Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining},
pages = {86--94},
url = {https://utexas.box.com/v/braylan-kdd21},
slides = {https://utexas.box.com/v/braylan-kdd21-slides},
source = {https://github.com/Praznat/annotationmodeling},
data = {https://github.com/Praznat/annotationmodeling},
techreport = {https://apps.cs.utexas.edu/apps/tech-reports/195281},
video = {https://www.youtube.com/watch?v=l8-Y_3qHrR0}
}
@inproceedings{gao-ijcai21,
title = {Human-AI Collaboration with Bandit Feedback},
author = {Ruijiang Gao and Maytal Saar-Tsechansky and Maria De-Arteaga and Ligong Han and Min Kyung Lee and Matthew Lease},
booktitle = {Proceedings of the 30th International Joint Conference on Artificial Intelligence (IJCAI)},
year = {2021},
pages = {1722--1728},
url = {https://utexas.box.com/v/gao-ijcai2021},
source = {https://github.com/ruijiang81/hai-blbf},
techreport = {https://arxiv.org/abs/2105.10614}
}
@inproceedings{hettiachchi21-workshop,
author = {Danula Hettiachchi and Mark Sanderson and Jorge Goncalves and Simo Hosio and Gabriella Kazai and Matthew Lease
and Mike Schaekermann and Emine Yilmaz},
title = {{Investigating and Mitigating Biases in Crowdsourced Data}},
booktitle = {Proceedings of the 24th ACM Conference on Computer Supported Cooperative Work (CSCW)},
year = {2021},
url = {https://utexas.box.com/v/hettiachchi-cscw21-workshop},
confurl = {https://sites.google.com/view/biases-in-crowdsourced-data},
pages = {331--334},
note = {Accepted Workshop.}
}
@article{danula-cscw21-pacm,
author = {Hettiachchi, Danula and Schaekermann, Mike and McKinney, Tristan J. and Lease, Matthew},
title = {{The Challenge of Variable Effort Crowdsourcing and How Visible Gold Can Help}},
year = {2021},
issue_date = {October 2021},
volume = {5},
url = {https://www.amazon.science/publications/the-challenge-of-variable-effort-crowdsourcing-and-how-visible-gold-can-help},
doi = {10.1145/3476073},
journal = {Proceedings of the ACM on Human-Computer Interaction},
articleno = {332},
url = {https://utexas.box.com/v/hettiachchi-cscw21},
techreport = {https://arxiv.org/pdf/2105.09457},
video = {https://www.youtube.com/watch?v=uN8syzhUnRU},
note = {{Article number 332, 26 pages. Presented at the 24th ACM Conference on Computer Supported Cooperative Work
(CSCW)}},
data = {https://github.com/danula/crowdsourcing-variable-effort}
}
@inproceedings{keswani-aies21,
title = {Towards Unbiased and Accurate Deferral to Multiple Experts},
booktitle = {Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society (AIES)},
author = {Vijay Keswani and Matthew Lease and Krishnaram Kenthapadi},
year = {2021},
url = {https://www.amazon.science/publications/towards-unbiased-and-accurate-deferral-to-multiple-experts},
poster = {https://www.aies-conference.com/2021/wp-content/posters/163_%20Towards%20Unbiased%20and%20Accurate%20Deferral%20to%20Multiple%20Experts.pdf},
techreport = {https://arxiv.org/abs/2102.13004},
source = {https://github.com/vijaykeswani/Deferral-To-Multiple-Experts},
data = {https://github.com/vijaykeswani/Deferral-To-Multiple-Experts},
video = {https://vimeo.com/553335196},
slides = {https://utexas.box.com/v/keswani-aies21-slides},
pages = {154--165},
note = {Additional \href{https://slideslive.com/38956059}{video}}
}
@inproceedings{steiger-chi21,
title = {The Psychological Well-Being of Content Moderators: The Emotional Labor of Commercial Moderation and Avenues for Improving Support},
author = {Miriah Steiger and Timir J.\ Bharucha and Sukrit Venkatagiri and Martin J.\ Riedl and Matthew Lease},
booktitle = {Proceedings of the ACM CHI Conference on Human Factors in Computing Systems},
year = {2021},
pages = {1--14},
techreport = {https://apps.cs.utexas.edu/apps/tech-reports/194782},
url = {../papers/steiger-chi21.pdf},
video = {https://www.youtube.com/watch?v=qRnrBDQo08w},
note = {[short \href{https://www.youtube.com/watch?v=5kHHzkBOHSs&list=PLqhXYFYmZ-Vc-oiiwfAcE46XlYVdHQkG_&index=153}{30-second preview video}]}
}
@inproceedings{lease-rabb21,
title = {Data Labeling Work in the AI Ecosystem and Opportunities for Improvement},
author = {Matthew Lease and Mohammad Hossein Jarrahi and Saiph Savage},
booktitle = {Rabb Symposium on Embedding AI in Society},
month = {February},
day = {18--19},
year = {2021},
address = {NC State University},
video = {https://youtu.be/PrOaYV_qX60},
slides = {https://docs.google.com/presentation/d/1V8_H3VpLRqmAJdZ-NUdAISnulBwP2o_tGlAhnv9pPS0/edit?usp=sharing},
confurl = {https://sites.google.com/view/ai-society-at-nc-state/conference?authuser=0},
note = {Presentation.}
}
@techreport{lease20-cstr,
title = {Promoting Psychological Wellness of Content Moderators},
author = {Matthew Lease and Miriah Steiger and Timir J. Bharucha and Martin J. Riedl and Sukrit Venkatagiri},
year = {2020},
institution = {University of Texas at Austin, Department of Computer Science},
number = {TR-20-02},
note = {June 1.},
url = {https://apps.cs.utexas.edu/apps/tech-reports/194782}
}
@mastersthesis{Li019,
author = {Qiwei Li},
title = {Clickbait and Emotional Language in Fake News},
school = {University of Texas at Austin, Department of Computer Science},
year = {2019},
type = {Bachelor's Thesis},
url = {../papers/li2019-thesis.pdf}
}
@article{roegiest-2019-facts-ir,
author = {Roegiest, Adam and Lipani, Aldo and Beutel, Alex and Olteanu, Alexandra and Lucic, Ana and Stoica, Ana-Andreea and Das, Anubrata and Biega, Asia and Voorn, Bart and Hauff, Claudia and Spina, Damiano and Lewis, David and Oard, Douglas W. and Yilmaz, Emine and Hasibi, Faegheh and Kazai, Gabriella and McDonald, Graham and Haned, Hinda and Ounis, Iadh and van der Linden, Ilse and Garcia-Gathright, Jean and Baan, Joris and Lau, Kamuela N. and Balog, Krisztian and de Rijke, Maarten and Sayed, Mahmoud and Panteli, Maria and Sanderson, Mark and Lease, Matthew and Ekstrand, Michael D. and Lahoti, Preethi and Kamishima, Toshihiro},
journal = {SIGIR Forum},
month = {December},
number = {2},
title = {FACTS-IR: Fairness, Accountability, Confidentiality, Transparency, and Safety in Information Retrieval},
volume = {53},
year = {2019},
pages = {20--43},
url = {../papers/roegiest-2019-facts-ir.pdf},
note = {Alexandra Olteanu, Jean Garcia-Gathright, Maarten de Rijke, and Michael D. Ekstrand, editors.}
}
@article{allan-swirl12,
title = {{Frontiers, Challenges, and Opportunities for Information Retrieval -- Report from SWIRL 2012, The Second Strategic Workshop on Information Retrieval in Lorne}},
author = {
James Allan and Jay Aslam and Leif Azzopardi and Nick Belkin and Pia Borlund and
Peter Bruza and Jamie Callan and Mark Carman and Charles L.A. Clarke and
Nick Craswell and W. Bruce Croft and J. Shane Culpepper and Fernando Diaz and
Susan Dumais and Nicola Ferro and Shlomo Geva and Julio Gonzalo and
David Hawking and Kalervo Jarvelin and Gareth Jones and Rosie Jones and
Jaap Kamps and Noriko Kando and Evangelos Kanoulas and Jussi Karlgren and
Diane Kelly and Matthew Lease and Jimmy Lin and Stefano Mizzaro and
Alistair Moffat and Vanessa Murdock and Douglas W. Oard and Maarten de Rijke and
Tetsuya Sakai and Mark Sanderson and Falk Scholer and Luo Si and James A. Thom and
Paul Thomas and Andrew Trotman and Andrew Turpin and Arjen P. de Vries and
William Webber and Xiuzhen Zhang and Yi Zhang
},
journal = {{SIGIR Forum}},
volume = {46},
number = {1},
pages = {2--32},
year = {2012},
publisher = {ACM},
url = {http://sigir.org/files/forum/2012J/2012j_sigirforum_A_allanSWIRL2012Report.pdf},
note = {James Allan, Bruce Croft, Alistair Moffat, and Mark Sanderson, editors.}
}
@article{aroyo2018reports,
title = {{Work in the Age of Intelligent Machines: Report of the Workshop Held at the Sixth AAAI Conference on Human Computation and Crowdsourcing}},
author = {Jeffrey V. Nickerson and Matthew Lease},
journal = {AI Magazine},
volume = {39},
number = {4},
pages = {60},
year = {2018},
publisher = {Association for the Advancement of Artificial Intelligence},
url = {https://ojs.aaai.org/index.php/aimagazine/article/view/2834/2741},
confurl = {https://waim.network/hcomp18},
note = {Workshop organized by Jeffrey V. Nickerson, Matthew Lease, Kevin Crowston, and Ingrid Erickson.}
}
@inproceedings{Bies06,
author = {Ann Bies and Stephanie Strassel and Haejoong Lee and Kazuaki Maeda and Seth Kulick and Yang Liu and Mary Harper and Matthew Lease},
title = {Linguistic Resources for Speech Parsing},
booktitle = {Fifth International Conference on Language Resources and Evaluation (LREC'06)},
year = 2006,
address = {Genoa, Italy},
url = {http://papers.ldc.upenn.edu/LREC2006/LinguisticResourcesForSpeechParsing.pdf}
}
@inproceedings{Braylan-web2020,
title = {{Modeling and Aggregation of Complex Annotations via Annotation Distances}},
author = {Alexander Braylan and Matthew Lease},
booktitle = {{Proceedings of the Web Conference}},
year = {2020},
pages = {1807-1818},
url = {../papers/braylan_web2020.pdf},
source = {https://github.com/Praznat/annotationmodeling},
data = {https://github.com/Praznat/annotationmodeling},
slides = {https://www.slideshare.net/AlexanderBraylan/modeling-and-aggregation-of-complex-annotations-238933764},
video = {https://www.youtube.com/watch?v=Asa_Bswlvfo}
}
@conference{Braylan-annonlp19,
author = {Alexander Braylan and Matthew Lease},
title = {Distance-based Consensus Modeling for Complex Annotations},
booktitle = {{Workshop on Aggregating and Analysing Crowdsourced Annotations for NLP (AnnoNLP) at the EMNLP-IJCNLP Conference}},
year = {2019},
confurl = {http://dali.eecs.qmul.ac.uk/annonlp},
url = {../papers/braylan-annonlp19.pdf},
slides = {../papers/alexbraylan_AnnoNLP.pptx},
source = {https://github.com/Praznat/annotationmodeling},
note = {6 pages.}
}
@conference{Braylan-hcompdc19,
author = {Alexander Braylan and Matthew Lease},
title = {Modeling Complex Annotations},
booktitle = {{Doctoral Consortium at the AAAI Human Computation and Crowdsourcing (HCOMP) Conference}},
year = {2019},
confurl = {https://www.humancomputation.com/2019/attend.html#dc},
url = {../papers/braylan-hcompdc19.pdf},
slides = {../papers/alexbraylan_HCOMP_DC.pptx},
source = {https://github.com/Praznat/annotationmodeling},
note = {4 pages.}
}
@inproceedings{Broussard10,
author = {Ramona Broussard and Yongyi Zhou and Matthew Lease},
title = {Mobile Phone Search for Library Catalogs},
booktitle = {Proceedings of the 73rd Annual Meeting of the American Society for Information Science and Technology (ASIS\&T)},
year = 2010,
url = {../papers/broussard-asist10.pdf},
note = {4 pages.},
video = {../papers/zhou-brossard-mut.avi},
source = {http://www.ischool.utexas.edu/~ramona/documents/mobileUT.zip},
slides = {http://www.ischool.utexas.edu/~ramona/documents/broussard-asist-web.ppt}
}
@inproceedings{Broussard10-2,
author = {Ramona Broussard and Yongyi Zhou and Matthew Lease},
title = {University of Texas Mobile Library Search},
booktitle = {Proceedings of the 73rd Annual Meeting of the American Society for Information Science and Technology (ASIS\&T)},
year = 2010,
video = {../papers/zhou-brossard-mut.avi},
url = {../papers/broussard-asist10-poster.pdf},
note = {2 pages.}
}
@inproceedings{Buckley10-notebook,
author = {Chris Buckley and Matthew Lease and Mark D. Smucker},
title = {{Overview of the TREC 2010 Relevance Feedback Track (Notebook)}},
booktitle = {{The Nineteenth Text Retrieval Conference (TREC) Notebook}},
institute = {{National Institute of Standards and Technology (NIST)}},
year = {2010},
url = {../papers/trec-notebook-2010.pdf},
note = {4 pages.}
}
@inproceedings{Cartright09,
author = {Marc Cartright and Jangwon Seo and Matthew Lease},
title = {{UMass Amherst and UT Austin at the TREC'09 Relevance Feedback Track}},
booktitle = {Proceedings of the 18th Text Retrieval Conference (TREC'09)},
institute = {National Institute of Standards and Technology},
year = 2010,
url = {http://trec.nist.gov/pubs/trec18/papers/umass-amhearst.RF.pdf},
note = {10 pages.}
}
@article{Carvalho10-forum,
author = {Vitor Carvalho and Matthew Lease and Emine Yilmaz},
title = {Crowdsourcing for Search Evaluation},
journal = {ACM SIGIR Forum},
volume = 44,
number = 2,
month = {December},
year = 2010,
pages = {17--22},
url = {http://sigir.org/files/forum/2010D/sigirwksp/2010d_sigirforum_carvalho.pdf},
confurl = {http://www.sigir.org/forum/2010D-TOC.html}
}
@inproceedings{Cheng15-iconf,
author = {James Cheng and Monisha Manoharan and Matthew Lease and Yan Zhang},
title = {{Is there a Doctor in the Crowd? Diagnosis Needed! (for less than \$5)}},
booktitle = {{Proceedings of the iConference}},
url = {http://www.ischool.utexas.edu/~ml/papers/cheng-iconf15.pdf},
year = {2015},
note = {16 pages}
}
@inproceedings{Dang18-ci,
title = {{Toward Safer Crowdsourced Content Moderation}},
author = {Brandon Dang and Martin J. Riedl and Matthew Lease},
booktitle = {{6th ACM Collective Intelligence Conference}},
year = {2018},
url = {https://arxiv.org/pdf/1804.10999v1},
confurl = {https://ci.acm.org/2018/},
source = {https://github.com/budang/content-moderation},
demo = {http://ai.ischool.utexas.edu/CM/demo/},
blog = {https://humancomputation.com/blog/?p=9932},
slides = {https://www.slideshare.net/mattlease/but-who-protects-the-moderators},
note = {5 pages. Peer-reviewed, non-archival. Extended version at AAAI HCOMP 2018. Demo URL updated since publication.}
}
@inproceedings{Dang18-hcomp,
title = {{But Who Protects the Moderators? The Case of Crowdsourced Image Moderation}},
author = {Brandon Dang and Martin J. Riedl and Matthew Lease},
booktitle = {{6th AAAI Conference on Human Computation and Crowdsourcing (HCOMP): Works-in-Progress Track}},
year = {2018},
confurl = {https://www.humancomputation.com/2018/},
demo = {http://ai.ischool.utexas.edu/CM/demo/},
url = {https://arxiv.org/pdf/1804.10999},
blog = {https://humancomputation.com/blog/?p=9932},
source = {https://github.com/budang/content-moderation},
slides = {https://www.slideshare.net/mattlease/but-who-protects-the-moderators},
note = {5 pages, peer-reviewed, non-archival. Demo URL updated since publication.}
}
@inproceedings{Dang16-hcomp,
title = {{MmmTurkey: A Crowdsourcing Framework for Deploying Tasks and Recording Worker Behavior on Amazon Mechanical Turk}},
author = {Brandon Dang and Miles Hutson and Matthew Lease},
booktitle = {{Proceedings of the 4th AAAI Conference on Human Computation and Crowdsourcing (HCOMP): Works-in-Progress Track}},
year = {2016},
url = {http://arxiv.org/abs/1609.00945},
source = {https://github.com/CuriousG102/turkey},
note = {3 pages. arXiv:1609.00945}
}
@conference{Das20-hcomp,
author = {Anubrata Das and Brandon Dang and Matthew Lease},
title = {Fast, Accurate, and Healthier: Interactive Blurring Helps Moderators Reduce Exposure to Harmful Content},
booktitle = {{Proceedings of the 8th AAAI Conference on Human Computation and Crowdsourcing (HCOMP)}},
year = {2020},
source = {https://github.com/budang/content-moderation},
demo = {http://ai.ischool.utexas.edu/CM/demo/},
blog = {https://humancomputation.com/blog/?p=9932},
video = {https://youtu.be/4ZIiGIkYdNA},
url = {../papers/das_hcomp20.pdf},
slides = {https://www.slideshare.net/mattlease/adventures-in-crowdsourcing-toward-safer-content-moderation-better-supporting-complex-annotation-tasks},
pages = {33--42}
}
@techreport{Das19-arxiv,
title = {{A Conceptual Framework for Evaluating Fairness in Search}},
author = {Anubrata Das and Matthew Lease},
institution = {University of Texas at Austin},
month = {July},
day = {22},
year = {2019},
url = {http://arxiv.org/abs/arXiv:1907.09328},
note = {arXiv:1907.09328}
}
@conference{Das19-factsir,
author = {Anubrata Das and Kunjan Mehta and Matthew Lease},
title = {CobWeb: A Research Prototype for Exploring User Bias in Political Fact-Checking},
booktitle = {{ACM SIGIR Workshop on Fairness, Accountability, Confidentiality, Transparency, and Safety in Information Retrieval (FACTS-IR)}},
year = {2019},
confurl = {https://facts-ir-2019.hotcrp.com},
url = {https://arxiv.org/abs/1907.03718},
note = {8 pages.}
}
@inproceedings{Goyal18-hcomp,
title = {{Your Behavior Signals Your Reliability: Modeling Crowd Behavioral Traces to Ensure Quality Relevance Annotations}},
author = {Tanya Goyal and Tyler McDonnell and Mucahid Kutlu and Tamer Elsayed and Matthew Lease},
booktitle = {Proceedings of the 6th AAAI Conference on Human Computation and Crowdsourcing (HCOMP)},
year = 2018,
data = {http://ai.ischool.utexas.edu/webcrowd25k/},
source = {https://github.com/budang/turkey-lite},
url = {../papers/goyal-hcomp18},
pages = {41--49},
slides = {https://www.slideshare.net/mattlease/your-behavior-signals-your-reliability-modeling-crowd-behavioral-traces-to-ensure-quality-relevance-annotations},
note = {Online version here includes corrections to official version from proceedings.}
}
@inproceedings{Grady10,
author = {Grady, Catherine and Lease, Matthew},
title = {Crowdsourcing Document Relevance Assessment with Mechanical Turk},
booktitle = {Proceedings of the NAACL HLT 2010 Workshop on Creating Speech and Language Data with Amazon's Mechanical Turk},
month = {June},
year = {2010},
address = {Los Angeles},
publisher = {Association for Computational Linguistics},
pages = {172--179},
url = {http://www.aclweb.org/anthology/W10-0727},
data = {http://www.ischool.utexas.edu/~ml/data/naacl-amt-2010.zip},
confurl = {http://sites.google.com/site/amtworkshop2010}
}
@conference{Guo11-enir,
author = {Lu Guo and Matthew Lease},
title = {Personalizing Local Search with Twitter},
booktitle = {{Workshop on Enriching Information Retrieval (ENIR) at the 34th Annual ACM SIGIR Conference}},
year = {2011},
url = {http://www.ischool.utexas.edu/~ml/papers/guo-enir11.pdf},
note = {2 pages, Oral presentation.},
confurl = {http://www.select.cs.cmu.edu/meetings/enir2011},
video = {http://www.ischool.utexas.edu/~ml/papers/guo-enir11.wmv},
source = {http://www.ischool.utexas.edu/~ml/papers/guo-enir11-sourcecode.zip}
}
@inproceedings{Gupta19-ecir,
author = {Soumyajit Gupta and Mucahid Kutlu and Vivek Khetan and Matthew Lease},
title = {{Correlation, Prediction and Ranking of Evaluation Metrics in Information Retrieval}},
booktitle = {{Proceedings of the 41st European Conference on Information Retrieval (ECIR)}},
year = {2019},
pages = {636--651},
techreport = {http://arxiv.org/abs/1802.00323},
slides = {../papers/gupta-ecir19-slides.pdf},
news = {https://www.ischool.utexas.edu/news/best-student-paper-awarded-ischool-associate-professor-and-student-collaborators},
url = {../papers/gupta-ecir19.pdf},
source = {https://github.com/smjtgupta/IR-corr-pred-rank},
data = {https://github.com/smjtgupta/IR-corr-pred-rank},
note = {\textbf{Best Student Paper award}.}
}
@inproceedings{hale2006pcfgs,
title = {PCFGs with syntactic and prosodic indicators of speech repairs},
author = {Hale, John and Shafran, Izhak and Yung, Lisa and Dorr, Bonnie and Harper, Mary and Krasnyanskaya, Anna and Lease, Matthew and Liu, Yang and Roark, Brian and Snover, Matthew and others},
booktitle = {Proceedings of the 21st International Conference on Computational Linguistics and the 44th annual meeting of the Association for Computational Linguistics},
pages = {161--168},
year = {2006},
organization = {Association for Computational Linguistics},
url = {http://www.aclweb.org/anthology/P06-1021.pdf}
}
@techreport{Harper05,
title = {{Parsing Speech and Structural Event Detection (PASSED): CLSP Summer Workshop Final Report}},
author = {Mary Harper and Bonnie Dorr and John Hale and Brian Roark and Izhak Shafran and Matthew Lease and Yang
Liu
and Matthew Snover and Lisa Yunge and Anna Krasnyanskayai and Robin Stewart},
institution = {Johns Hopkins University},
year = {2005},
confurl = {http://old-site.clsp.jhu.edu/ws2005/groups/eventdetect},
slides = {http://old-site.clsp.jhu.edu/ws2005/groups/eventdetect/documents/FinalPresentation.pdf},
url = {../papers/clsp05-finalreport.pdf}
}
@inproceedings{Johnson04-rtf,
author = {Mark Johnson and Eugene Charniak and Matthew Lease},
title = {An Improved Model For Recognizing Disfluencies in Conversational Speech},
booktitle = {Rich Transcription 2004 Fall Workshop (RT-04F)},
year = {2004},
url = {../papers/rt04-paper.pdf}
}
@article{josephy2014workshops,
title = {Crowdsourcing at Scale: Workshop Held at the First AAAI Conference on Human Computation and Crowdsourcing},
author = {Josephy, Tatiana and Lease, Matthew and Paritosh, Praveen},
journal = {AI Magazine},
volume = {35},
number = {2},
pages = {76--77},
year = {2014},
url = {../papers/CrowdScale2013.pdf},
confurl = {https://sites.google.com/site/crowdscale2013/}
}
@phdthesis{Jung-thesis15,
author = {Hyun Joon Jung},
title = {{Temporal Modeling Crowd Work for Quality Assurance in Crowdsourcing}},
school = {School of Information, University of Texas at Austin},
month = {December},
year = 2015,
url = {https://repositories.lib.utexas.edu/bitstream/handle/2152/33261/JUNG-DISSERTATION-2015.pdf}
}
@inproceedings{Jung15-hcomp,
author = {Hyun Joon Jung and Matthew Lease},
title = {{Modeling Temporal Crowd Work Quality with Limited Supervision}},
booktitle = {{Proceedings of the 3rd AAAI Conference on Human Computation (HCOMP)}},
year = {2015},
pages = {83--91},
url = {../papers/jung-hcomp15.pdf}
}
@inproceedings{Jung15-crowdml,
author = {Hyun Joon Jung and Matthew Lease},
title = {{Forecasting Crowd Work Quality via Multi-dimensional Features of Workers}},
booktitle = {{ICML Workshop on Crowdsourcing and Machine Learning (CrowdML)}},
year = {2015},
url = {../papers/jung-crowdml15.pdf},
note = {10 pages.}
}
@inproceedings{Jung15-ecir,
author = {Hyun Joon Jung and Matthew Lease},
title = {{A Discriminative Approach to Predicting Assessor Accuracy}},
booktitle = {{Proceedings of the 37th European Conference on Information Retrieval (ECIR)}},
year = {2015},
pages = {159-171},
url = {../papers/ecir2015_hjung.pdf},
news = {https://www.ischool.utexas.edu/news/hyun-joon-jung-receives-2015-samsung-human-tech-paper-award},
note = {{\bf Samsung Human-Tech Paper Award: Silver Prize in Computer Science.}}
}
@inproceedings{Jung14-hcomp,
author = {Hyun Joon Jung and Yubin Park and Matthew Lease},
title = {{Predicting Next Label Quality: A Time-Series Model of Crowdwork}},
booktitle = {{Proceedings of the 2nd AAAI Conference on Human Computation (HCOMP)}},
year = {2014},
pages = {87--95},
url = {../papers/hcomp14-jung.pdf}
}
@inproceedings{Jung14-www,
author = {Hyun Joon Jung},
title = {{Quality Assurance in Crowdsourcing via Matrix Factorization based Task Routing}},
booktitle = {{Proceedings of World Wide Web (WWW) Ph.D. Symposium, Companion Publication}},
year = {2014},
pages = {3--8},
confurl = {http://www2014.kr/calls/call-for-ph-d-symposium/},
url = {../papers/jung-www14.pdf}
}
@techreport{Jung13-arxiv,
title = {Crowdsourced Task Routing via Matrix Factorization},
author = {Hyun Joon Jung and Matthew Lease},
institution = {University of Texas at Austin},
month = {October},
day = {18},
year = {2013},
url = {http://arxiv.org/abs/1310.5142},
note = {arXiv:1310.5142}
}
@techreport{Jung12-arxiv,
title = {Evaluating Classifiers Without Expert Labels},
author = {Hyun Joon Jung and Matthew Lease},
institution = {University of Texas at Austin},
month = {December},
day = {5},
year = {2012},
url = {http://arxiv.org/pdf/1212.0960v1},
note = {arXiv:1212.0960}
}
@inproceedings{Jung13-trec,
author = {Hyun Joon Jung and Matthew Lease},
title = {UT Austin in the TREC 2012 Crowdsourcing Track's Image Relevance Assessment Task},
booktitle = {{Proceedings of the 21st NIST Text Retrieval Conference (TREC)}},
url = {http://trec.nist.gov/pubs/trec21/papers/UTAustin.crowd.final.pdf},
year = {2013},
note = {12 pages}
}
@inproceedings{Jung11-hcomp,
author = {Hyun Joon Jung and Matthew Lease},
title = {{Improving Consensus Accuracy via Z-score and Weighted Voting}},
booktitle = {{Proceedings of the 3rd Human Computation Workshop (HCOMP) at AAAI}},
year = {2011},
pages = {88--90},
url = {http://www.ischool.utexas.edu/~ml/papers/jung-hcomp11.pdf},
blog = {https://humancomputation.com/blog/?p=1206},
confurl = {http://humancomputation.com/2011}
}
@inproceedings{Jung12-hcomp,
author = {Hyun Joon Jung and Matthew Lease},
title = {{Improving Quality of Crowdsourced Labels via Probabilistic Matrix Factorization}},
booktitle = {{Proceedings of the 4th Human Computation Workshop (HCOMP) at AAAI}},
year = {2012},
pages = {101--106},
confurl = {http://www.humancomputation.com/2012},
url = {https://www.aaai.org/ocs/index.php/WS/AAAIW12/paper/viewFile/5258/5609}
}
@inproceedings{Jung12-sigir,
author = {Hyun Joon Jung and Matthew Lease},
title = {{Inferring Missing Relevance Judgments from Crowd Workers via Probabilistic Matrix Factorization}},
booktitle = {{Proceedings of the 35th international ACM SIGIR conference on Research and Development in Information Retrieval}},
year = {2012},
pages = {1095--1096},
url = {../papers/jung-sigir12.pdf}
}
@inproceedings{kahn-EtAl:2005:HLTEMNLP,
author = {Kahn, Jeremy G. and Lease, Matthew and Charniak, Eugene and Johnson, Mark and Ostendorf, Mari},
title = {Effective Use of Prosody in Parsing Conversational Speech},
booktitle = {Proceedings of Human Language Technology Conference and Conference on Empirical Methods in Natural Language Processing (EMNLP'05)},
month = {October},
year = {2005},
address = {Vancouver, British Columbia, Canada},
publisher = {Association for Computational Linguistics},
pages = {233--240},
url = {http://cs.brown.edu/research/pubs/pdfs/2005/Kahn-2005-EUP.pdf}
}
@inproceedings{Kittur-cscw13,
author = {Aniket Kittur and Jeffrey V.\ Nickerson and Michael S. Bernstein and Elizabeth Gerber and Aaron Shaw and John Zimmerman and Matthew Lease and John J. Horton},
title = {The Future of Crowd Work},
booktitle = {Proceedings of the ACM Conference on Computer Supported Cooperative Work (CSCW)},
pages = {1301--1318},
month = {February},
year = {2013},
url = {http://wtf.tw/ref/kittur.pdf},
blog = {https://humancomputation.com/blog/?p=4799},
note = {Social Science Research Network (SSRN) ID: 2190946}
}
@inproceedings{Klinger11-asist,
author = {Jorn Klinger and Matthew Lease},
title = {Enabling Trust in Crowd Labor Relations through Identity Sharing},
booktitle = {Proceedings of the 74th Annual Meeting of the American Society for Information Science and Technology (ASIS\&T)},
year = 2011,
note = {4 pages.},
url = {../papers/klinger-asist11.pdf},
confurl = {http://www.asis.org/asist2011}
}
@inproceedings{Kovashka10-crowdconf,
author = {Kovashka, Adriana and Lease, Matthew},
title = {Human and Machine Detection of Stylistic Similarity in Art},
booktitle = {Proceedings of the 1st Annual Conference on the Future of Distributed Work (CrowdConf)},
month = {September},
year = {2010},
address = {San Francisco},
url = {../papers/kovashka-crowdconf10.pdf},
confurl = {https://web.archive.org/web/20101219063421/http://crowdconf.com:80/papers.html},
note = {9 pages.}
}
@mastersthesis{Kumar-thesis12,
author = {Abhimanu Kumar},
title = {{Supervised language models for temporal resolution of text in absence of explicit temporal cues}},
school = {Department of Computer Science, University of Texas at Austin},
month = {May},
year = 2012,
note = {Supervisor: Joydeep Ghosh. Readers: Jason Baldridge and Matthew Lease.},
url = {http://repositories.lib.utexas.edu/handle/2152/23581}
}
@techreport{Kumar12-arxiv,
title = {Dating Texts without Temporal Cues},
author = {Abhimanu Kumar and Jason Baldridge and Matthew Lease and Joydeep Ghosh},
institution = {University of Texas at Austin},
month = {November},
day = {9},
year = {2012},
url = {http://arxiv.org/pdf/1211.2290v1},
note = {arXiv:1211.2290}
}
@inproceedings{Kumar11-sigir,
author = {Abhimanu Kumar and Matthew Lease},
title = {Learning to Rank From a Noisy Crowd},
booktitle = {{Proceedings of the 34th Annual ACM SIGIR Conference}},
year = {2011},
url = {http://www.ischool.utexas.edu/~ml/papers/kumar-sigir11.pdf},
pages = {1221-1222},
note = {Separately reviewed and accepted for encore presentation at the 3rd Human Computation Workshop (HCOMP) at AAAI 2011. Appears in SIGIR proceedings only.}
}
@inproceedings{Kumar11-cikm,
title = {Supervised Language Modeling for Temporal Resolution of Texts},
author = {Abhimanu Kumar and Matthew Lease and Jason Baldridge},
booktitle = {{Proceedings of the 20th ACM Conference on Information and Knowledge Management (CIKM)}},
year = {2011},
pages = {2069-2072},
url = {http://www.ischool.utexas.edu/~ml/papers/kumar-cikm11.pdf}
}
@inproceedings{Kumar11-csdm,
author = {Abhimanu Kumar and Matthew Lease},
title = {Modeling Annotator Accuracies for Supervised Learning},
booktitle = {{Proceedings of the Workshop on Crowdsourcing for Search and Data Mining (CSDM) at the Fourth ACM International Conference on Web Search and Data Mining (WSDM)}},
pages = {19--22},
month = {February},
day = {9},
year = {2011},
address = {Hong Kong, China},
url = {http://ai.ischool.utexas.edu/csdm2011/proceedings/csdm2011_kumar.pdf},
slides = {http://ai.ischool.utexas.edu/csdm2011/slides/kumar_csdm11_slides.pdf},
confurl = {http://ai.ischool.utexas.edu/csdm2011/proceedings.html}
}
@article{Kutlu20-jair,
author = {Mucahid Kutlu and Tyler McDonnell and Tamer Elsayed and Matthew Lease},
title = {{Annotator Rationales for Labeling Tasks in Crowdsourcing}},
journal = {Journal of Artificial Intelligence Research (JAIR)},
year = {2020},
volume = 69,
pages = {143--189},
data = {http://github.com/tylermcdonnell/WhyIsThatRelevant},
blog = {https://humancomputation.com/blog/?p=9483},
note = {{\bf Award Winning Papers Track}.},
url = {../papers/kutlu_jair20.pdf},
confurl = {https://www.jair.org/index.php/jair/awardedPapers}
}
@inproceedings{Kutlu18-cikm,
title = {When Rank Order isn't Enough: New Statistical-Significance-Aware Correlation Measures},
author = {Mucahid Kutlu and Tamer Elsayed and Maram Hasanain and Matthew Lease},
booktitle = {{Proceedings of the 27th ACM International Conference on Information and Knowledge Management (CIKM)}},
year = {2018},
url = {../papers/kutlu-cikm18.pdf},
pages = {397--406}
}
@inproceedings{Kutlu18-sigir,
author = {Mucahid Kutlu and Tyler McDonnell and Yassmine Barkallah and Tamer Elsayed and Matthew Lease},
title = {{Crowd vs.\ Expert: What Can Rationales behind Relevance Judgments Tell Us About Assessor Disagreement?}},
booktitle = {Proceedings of the 41st international ACM SIGIR conference on Research and development in Information Retrieval},
year = {2018},
data = {http://ai.ischool.utexas.edu/webcrowd25k/},
url = {../papers/kutlu-sigir18.pdf},
pages = {805--814}
}
@article{Kutlu-ipm18,
title = {{Learning to Effectively Select Topics For Information Retrieval Test Collections}},
author = {Mucahid Kutlu and Tamer Elsayed and Matthew Lease},
journal = {Information Processing and Management (IPM)},
year = {2018},
volume = 54,
number = 1,
pages = {37--59},
publisher = {Elsevier},
doi = {10.1016/j.ipm.2017.09.002},
techreport = {https://arxiv.org/abs/1701.07810},
url = {../papers/kutlu-ipm18.pdf}
}
@inproceedings{Kutlu18-desires,
title = {{Mix and Match: Collaborative Expert-Crowd Judging for Building Test Collections Accurately and Affordably}},
author = {Mucahid Kutlu and Tyler McDonnell and Aashish Sheshadri and Tamer Elsayed and Matthew Lease},
booktitle = {Proceedings of the 1st Biannual Conference on the Design of Experimental Search \& Information REtrieval Systems (DESIRES)},
year = {2018},
pages = {42--46},
confurl = {http://desires.dei.unipd.it/},
techreport = {https://arxiv.org/abs/1806.00755},
slides = {https://www.slideshare.net/mattlease/mix-and-match-collaborative-expertcrowd-judging-for-building-test-collections-accurately-affordably},
url = {../papers/kutlu-desires18.pdf},
note = {CEUR Workshop Proceedings Vol-2167, \url{http://ceur-ws.org/Vol-2167}.}
}
@inproceedings{Lease20-ideas,
title = {{Designing Assistive AI Technologies to Support Human Judging of Information Reliability}},
author = {Matthew Lease},
booktitle = {Virtual Conference on Social-Cybersecurity in Times of Crisis and Change, Center for Informed Democracy & Social-Cybersecurity (IDeaS), Carnegie Mellon University},
year = {2020},
confurl = {https://www.cmu.edu/ideas-social-cybersecurity/events/conference-archive/archive-conference-2020.html},
slides = {https://www.slideshare.net/mattlease/},
url = {../papers/lease-ideas20.pdf},
note = {Presentation abstract.}
}
@inproceedings{Lease18-desires,
title = {{Fact Checking and Information Retrieval}},
author = {Matthew Lease},
booktitle = {Proceedings of the 1st Biannual Conference on the Design of Experimental Search \& Information REtrieval Systems (DESIRES)},
year = {2018},
pages = {97-98},
confurl = {http://desires.dei.unipd.it/},
slides = {https://www.slideshare.net/mattlease/fact-checking-information-retrieval},
url = {../papers/lease-desires18.pdf},
note = {CEUR Workshop Proceedings Vol-2167, \url{http://ceur-ws.org/Vol-2167}.}
}
@inproceedings{Lease16-nieuw,
title = {{Crowdsourcing for Success: Motivations, Design, \& Ethics}},
author = {Matthew Lease},
booktitle = {Workshop on Novel Incentives and Engineering Unique Workflows (NIEUW), organized by the Linguistic Data Consortium (LDC)},
year = {2016},
confurl = {https://www.ldc.upenn.edu/communications/workshops/nieuw-workshop},
url = {../papers/lease-nieuw16.pdf},
note = {2 pages.}
}
@inproceedings{Lease16-medir,
title = {{Systematic Review is e-Discovery in Doctor's Clothing}},
author = {Matthew Lease and Gordon V.\ Cormack and Nguyen, An Thanh and Thomas A.\ Trikalinos and Byron C.\ Wallace},
booktitle = {Proceedings of the Medical Information Retrieval (MedIR) Workshop at the 39th International ACM SIGIR Conference on Research and Development in Information Retrieval},
year = {2016},
slides = {https://www.slideshare.net/mattlease/systematic-review-is-ediscovery-in-doctors-clothing},
url = {../papers/lease-medir16.pdf},
note = {2 pages.}
}
@inproceedings{Lease18-ency,
author = {Lease, Matthew and Alonso, Omar},
editor = {Alhajj, Reda and Rokne, Jon},
title = {Crowdsourcing and Human Computation: Introduction},
booktitle = {Encyclopedia of Social Network Analysis and Mining},
year = {2018},
publisher = {Springer New York},
address = {New York, NY},
pages = {499--510},
isbn = {978-1-4939-7131-2},
doi = {10.1007/978-1-4939-7131-2_107},
url = {https://utexas.box.com/v/lease-esnam18}
}
@inproceedings{Lease14-ency,
author = {Lease, Matthew and Alonso, Omar},
editor = {Alhajj, Reda and Rokne, Jon},
title = {Crowdsourcing and Human Computation, Introduction},
booktitle = {Encyclopedia of Social Network Analysis and Mining},
year = {2014},
publisher = {Springer New York},
address = {New York, NY},
pages = {304--315},
isbn = {978-1-4614-6170-8},
doi = {10.1007/978-1-4614-6170-8_107},
url = {https://utexas.box.com/v/lease-esnam14}
}
@article{Lease13-irj,
author = {Matthew Lease and Emine Yilmaz},
title = {{Crowdsourcing for Information Retrieval: Introduction to the Special Issue}},
journal = {Information Retrieval (Springer)},
volume = 16,
number = 2,
month = {April},
year = 2013,
pages = {91-100},
url = {http://link.springer.com/content/pdf/10.1007%2Fs10791-013-9222-7.pdf},
confurl = {https://link.springer.com/journal/10791/volumes-and-issues/16-2}
}
@book{Lease13-irj-edited,
editor = {Matthew Lease and Emine Yilmaz},
title = {Crowdsourcing for Information Retrieval (Special Issue)},
publisher = {{\em Information Retrieval (Springer)}},
url = {https://link.springer.com/journal/10791/volumes-and-issues/16-2},
month = {April},
year = 2013,
note = {16(2):91--305}
}
@techreport{Lease-ssrn13,
author = {Matthew Lease and Jessica Hullman and Jeffrey P.\ Bigham and Michael S.\ Bernstein and Juho Kim and Walter S. Lasecki and Saeideh Bakhshi and Tanushree Mitra and Robert C.\ Miller},
title = {Mechanical Turk is Not Anonymous},
institution = {Social Science Research Network (SSRN)},
month = {March 6,},
year = {2013},
url = {http://wtf.tw/ref/lease.pdf},
blog = {https://humancomputation.com/blog/?p=5177},
note = {SSRN ID: 2228728}
}
@inproceedings{Lease11-trec,
author = {Matthew Lease and Gabriella Kazai},
title = {{Overview of the TREC 2011 Crowdsourcing Track (Conference
Notebook)}},
booktitle = {20th Text Retrieval Conference (TREC)},
year = 2011,
note = {10 pages.}
}
@book{crowdscale2013,
editor = {Matthew Lease and Praveen Paritosh and Tatiana Josephy},
title = {{Proceedings of the AAAI Human Computation Workshop on Crowdsourcing at Scale (CrowdScale)}},
month = {November},
day = {9},
year = {2013},
address = {Palm Springs, CA},
publisher = {Online},
confurl = {https://sites.google.com/site/crowdscale2013/workshop-agenda/accepted-papers},
note = {36 pages.}
}
@book{Lease-CSE2010,
editor = {Matthew Lease and Vitor Carvalho and Emine Yilmaz},
title = {{Proceedings of the ACM SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (CSE
2010)}},
month = {July},
day = {23},
year = {2010},
publisher = {Online},
address = {Geneva, Switzerland},
url = {http://ai.ischool.utexas.edu/cse2010/materials/CSE2010-Proceedings.pdf},
confurl = {http://ai.ischool.utexas.edu/cse2010},
note = {42 pages.}
}
@inproceedings{Lease11-ijcnlp,
author = {Matthew Lease},
title = {{Crowd Computing: Opportunities and Challenges}},
booktitle = {{Keynote at the 5th International Joint Conference on Natural Language Processing (IJCNLP)}},
month = {November},
day = {10},
year = {2011},
address = {Chiang Mai, Thailand},
confurl = {http://www.ijcnlp2011.org},
slides = {http://www.slideshare.net/mattlease/crowd-computing-opportunities-challenges-ijcnlp-2011-keynote}
}
@inproceedings{Lease11-hcomp,
author = {Matthew Lease},
title = {{On Quality Control and Machine Learning in Crowdsourcing}},
booktitle = {{Proceedings of the 3rd Human Computation Workshop (HCOMP) at
AAAI}},
pages = {97--102},
year = {2011},
url = {http://www.ischool.utexas.edu/~ml/papers/lease-hcomp11.pdf},
confurl = {http://www.humancomputation.com},
note = {{Separately refereed and accepted for encore presentation at the AAAI Spring Sym
posium 2012: Wisdom of the Crowd.}}
}
@book{Lease-CIR2011,
editor = {Matthew Lease and Emine Yilmaz and Alexander Sorokin and Vaughn Hester},
title = {{Proceedings of the 2nd Workshop on Crowdsourcing for Information Retrieval at the 34th ACM International Conference on Information Retrieval (SIGIR 2011)}},
month = {July},
day = {28},
year = {2011},
address = {Beijing, China},
publisher = {Online},
confurl = {https://sites.google.com/site/cir2011ws/proceedings},
url = {https://sites.google.com/site/cir2011ws/CIR2011Proceedings.pdf},
note = {65 pages.}
}
@book{Lease-CSDM2011,
editor = {Matthew Lease and Vitor Carvalho and Emine Yilmaz},
title = {{Proceedings of the Workshop on Crowdsourcing for Search and Data Mining (CSDM) at the Fourth ACM International Conference on Web Search and Data Mining (WSDM)}},
month = {February},
day = {9},
year = {2011},
address = {Hong Kong, China},
publisher = {Online},
confurl = {http://ai.ischool.utexas.edu/csdm2011/proceedings.html},
url = {http://ai.ischool.utexas.edu/csdm2011/proceedings/csdm2011_proceedings.pdf},
note = {38 pages.}
}
@article{Lease11-forum,
author = {Matthew Lease and Vitor Carvalho and Emine Yilmaz},
title = {Crowdsourcing for Search and Data Mining},
journal = {ACM SIGIR Forum},
volume = 45,
number = 1,
month = {June},
year = 2011,
pages = {18--24},
url = {http://sigir.org/files/forum/2011J/2011j_sigirforum_C_lease.pdf},
confurl = {http://www.sigir.org/forum/2010D-TOC.html}
}
@article{Lease11-forum2,
author = {Matthew Lease and Emine Yilmaz},
title = {Crowdsourcing for Information Retrieval},
journal = {ACM SIGIR Forum},
volume = 45,
number = 2,
month = {December},
year = 2011,
pages = {66--75},
url = {http://sigir.org/files/forum/2011D/workshops/2011d_sigirforum_lease.pdf}
}
@phdthesis{Lease09-phd,
author = {Matthew Lease},
title = {{Beyond Keywords: Finding Information More Accurately and Easily Using Natural Language}},
school = {Brown University Dept. of Computer Science},
month = {August 24,},
year = 2009,
note = {Degree conferred May 2010},
url = {http://cs.brown.edu/research/pubs/theses/phd/2010/lease.pdf}
}
@inproceedings{Lease09-trec,
author = {Matthew Lease},
title = {{Incorporating Relevance and Psuedo-relevance Feedback in the Markov Random Field Model: Brown at the TREC'08 Relevance
Feedback Track}},
booktitle = {Proceedings of the 17th Text Retrieval Conference (TREC'08)},
year = 2009,
note = {Best results in track. This paper supersedes an earlier version appearing in conference's Working Notes.},
data = {../data/trec08.tar.bz2},
url = {../papers/lease-trec08-proc.pdf}
}
@conference{Lease09-sigir,
title = {{An Improved Markov Random Field Model for Supporting Verbose Queries}},
author = {Matthew Lease},
booktitle = {Proceedings of the 32nd Annual ACM SIGIR Conference},
year = 2009,
pages = {476--483},
url = {../papers/lease-sigir09.pdf}
}
@conference{Lease09-ecir,
title = {{Regression Rank: Learning to Meet the Opportunity of Descriptive Queries}},
author = {Matthew Lease and James Allan and W. Bruce Croft},
booktitle = {Proceedings of the 31st European Conference on Information Retrieval (ECIR)},
year = {2009},
pages = {90--101},
url = {../papers/lease-ecir09.pdf},
data = {../data/mlease-ecir09-data.tar.bz2}
}
@inproceedings{Lease07-clef-proceedings,
author = {Matthew Lease and Eugene Charniak},
title = {A Dirichlet-smoothed Bigram Model for Retrieving Spontaneous Speech},
booktitle = {Advances in Multilingual and Multimodal Information Retrieval: 8th Workshop of the Cross-Language Evaluation Forum, CLEF 2007, Revised Selected Papers},
year = 2008,
isbn = {978-3-540-85759-4},
publisher = {Springer-Verlag},
series = {Lecture Notes in Computer Science},
volume = 5152,
url = {../papers/mlease-clef2007.pdf}
}
@inproceedings{Lease07-clef,
author = {Matthew Lease and Eugene Charniak},
title = {Brown at {CL-SR}'07: Retrieving Conversational Speech in {E}nglish and {C}zech},
booktitle = {Working Notes of the Cross-Language Evaluation Forum (CLEF): Cross-Language Speech Retrieval (CL-SR) track},
year = 2007,
note = {Corrected version},
url = {../papers/leaseCLEF2007-corrected.pdf}
}
@inproceedings{Lease07-pikm,
author = {Matthew Lease},
title = {Natural Language Processing for Information Retrieval: the time is ripe (again)},
booktitle = {Proceedings of the 1st Ph.D. Workshop at the ACM Conference on Information and Knowledge Management (PIKM)},
year = 2007,
note = {Best Paper award},
url = {../papers/lease-pikm07.pdf}
}
@article{Lease06-sap,
author = {Lease, Matthew and Johnson, Mark and Charniak, Eugene},
title = {Recognizing disfluencies in conversational speech},
journal = {IEEE Transactions on Audio, Speech and Language Processing},
volume = 14,
number = 5,
month = {September},
year = 2006,
pages = {1566-1573},
url = {../papers/lease06-sap.pdf}
}
@inproceedings{Lease06aaai,
author = {Matthew Lease and Eugene Charniak and Mark Johnson and David McClosky},
title = {A Look At Parsing and Its Applications},
booktitle = {Proceedings of the Twenty-First National Conference on Artificial Intelligence (AAAI-06)},
month = {16--20 July},
year = {2006},
url = {../papers/aaai06.pdf}
}
@inproceedings{lease-johnson:2006:HLT-NAACL06-Short,
author = {Lease, Matthew and Johnson, Mark},
title = {Early Deletion of Fillers In Processing Conversational Speech},
booktitle = {Proceedings of the Human Language Technology Conference of the NAACL (HLT-NAACL'06), Companion Volume: Short Papers},
month = {June},
year = {2006},
address = {New York City, USA},
publisher = {Association for Computational Linguistics},
pages = {73--76},
url = {../papers/mlease_naacl06.pdf},
note = {Version here corrects Table 2 in published version.}
}
@techreport{Lease05-tr,
title = {Parsing and Disfluency Modeling},
author = {Matthew Lease},
institution = {Brown University Department of Computer Science},
number = {CS-05-15},
year = 2005,
url = {http://www.cs.brown.edu/research/pubs/techreports/reports/CS-05-15.html}
}
@inproceedings{Lease05-icassp,
author = {Matthew Lease and Eugene Charniak and Mark Johnson},
title = {Parsing and its applications for conversational speech},
booktitle = {Proceedings of IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP'05)},
month = {March 18 - March 23},
year = {2005},
volume = 5,
pages = {961--964},
url = {../papers/lease05-icassp.pdf}
}
@inproceedings{Lease05-ijcnlp,
author = {Lease, Matthew and Charniak, Eugene},
title = {Parsing Biomedical Literature},
booktitle = {Proceedings of the 2nd International Joint Conference on Natural Language Processing (IJCNLP'05)},
editor = {R. Dale and K.-F. Wong and J. Su and O. Kwong},
month = {October 11 - October 13},
address = {Jeju Island, Korea},
year = {2005},
pages = {58 -- 69},
publisher = {Springer-Verlag},
series = {Lecture Notes in Computer Science (LNCS)},
volume = 3651,
url = {../papers/lease-ijcnlp05.pdf},
data = {../data/parser-treebank.tgz}
}
@conference{Lease03-icdcs,
title = {{SmartElevator}: Revitalizing A Legacy Device through Inexpensive Augmentation},
author = {Lease, Matthew and Eddon, Guy},
booktitle = {Proceedings of the IEEE 23rd International Conference on Distributed Computing Systems (ICDCS): 3rd International Workshop on Smart Appliances and Wearable Computing},
year = {2003},
pages = {254 -- 259},
url = {http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=1203563}
}
@inproceedings{Lease02,
title = {{Plan-Aware Behavioral Modeling}},
booktitle = {Adjunct Proceedings of 4th Intl. Conference on Ubiquitous Computing (UBICOMP)},
author = {Matthew Lease},
pages = {35--36},
year = {2002},
url = {http://www.ubicomp.org/ubicomp2002/ubicomp_adjunct_proceedings.pdf}
}
@techreport{Liu12-tr,
author = {Di Liu and Matthew Lease and Rebecca Kuipers and Randolph
Bias},
title = {Crowdsourcing for Usability Testing},
institution = {School of Information, University of Texas at Austin},
month = {March},
day = {7},
year = {2012},
url = {http://arxiv.org/pdf/1203.1468v2},
note = {arXiv:1203.1468}
}
@inproceedings{Liu12-asist,
author = {Di Liu and Randolph Bias and Matthew Lease and Rebecca Kuipers},
title = {Crowdsourcing for Usability Testing},
booktitle = {Proceedings of the 75th Annual Meeting of the American Society for Information Science and Technology (ASIS\&T)},
month = {October 28--31},
year = 2012,
techreport = {http://arxiv.org/abs/1203.1468},
url = {../papers/liu-asist12.pdf},
note = {10 pages.}
}
@inproceedings{Mankar17-hcomp,
title = {{Design Activism for Minimum Wage Crowd Work}},
author = {Akash Mankar and Riddhi J.\ Shah and Matthew Lease},
booktitle = {5th AAAI Conference on Human Computation and Crowdsourcing (HCOMP): Works-in-Progress Track},
year = 2017,
techreport = {https://arxiv.org/abs/1706.10097},
source = {http://github.com/akash-mankar/DesignActivism},
url = {../papers/mankar-hcomp17.pdf},
poster = {https://docs.google.com/presentation/d/1C1s0sGGf_w4e0VzQyy4nNrETR0_aDddqwqB1HlbJ9Pc/edit?usp=sharing},
note = {See extended technical report: arXiv 1706.10097.}
}
@inproceedings{McDonnell17-ijcai,
title = {{The Many Benefits of Annotator Rationales for Relevance Judgments}},
author = {Tyler McDonnell and Mucahid Kutlu and Tamer Elsayed and Matthew Lease},
booktitle = {Proceedings of the Twenty-Sixth International Joint Conference on Artificial Intelligence (IJCAI): Sister Conference Best Paper Track},
year = {2017},
url = {../papers/mcdonnell-ijcai17.pdf},
confurl = {https://ijcai-17.org/sister-conf.html},
data = {http://github.com/tylermcdonnell/WhyIsThatRelevant},
blog = {https://humancomputation.com/blog/?p=9483},
slides = {../papers/mcdonnell-hcomp16-slides.pdf},
pages = {4909--4913}
}
@inproceedings{McDonnell16-hcomp,
author = {Tyler McDonnell and Matthew Lease and Mucahid Kutlu and Tamer Elsayed},
title = {{Why Is That Relevant?\ Collecting Annotator Rationales for Relevance Judgments}},
booktitle = {{Proceedings of the 4th AAAI Conference on Human Computation and Crowdsourcing (HCOMP)}},
pages = {139--148},
year = {2016},
data = {http://github.com/tylermcdonnell/WhyIsThatRelevant},
news = {https://www.ischool.utexas.edu/news/professor-lease-wins-hcomp-2016-best-paper-award},
url = {../papers/mcdonnell-hcomp16.pdf},
blog = {https://humancomputation.com/blog/?p=9483},
slides = {../papers/mcdonnell-hcomp16-slides.pdf},
note = {{\bf Best Paper Award}}
}
@conference{Momtazi-ecdl10,
title = {{Effective Term Weighting for Sentence Retrieval}},
author = {Saeedeh Momtazi and Matthew Lease and Dietrich Klakow},
booktitle = {Proceedings of the 14th European Conference on Research and Advanced Technology for Digital Libraries (ECDL)},
year = {2010},
pages = {482--485},
series = {Lecture Notes in Computer Science (LNCS)},
publisher = {Springer-Verlag},
volume = 6273,
url = {../papers/SaeedehMomtazi-ECDL10.pdf}
}
@phdthesis{nguyen-dissertation20,
author = {Nguyen, An Thanh},
title = {{Probabilistic modeling with human factors in machine learning}},
school = {Computer Science, University of Texas at Austin},
month = {May},
year = 2020,
url = {https://repositories.lib.utexas.edu/bitstream/handle/2152/83387/NGUYEN-DISSERTATION-2020.pdf}
}
@inproceedings{Nguyen19-exss,
title = {{Mash: software tools for developing interactive and transparent machine learning systems}},
author = {Nguyen, An Thanh and Matthew Lease and Byron C.\ Wallace},
booktitle = {{Proceedings of ACM IUI Workshop on Explainable Smart Systems (ExSS)}},
year = {2019},
url = {../papers/nguyen-exss19.pdf},
note = {6 pages.}
}
@inproceedings{Nguyen19-iui,
title = {{Explainable Modeling of Annotations in Crowdsourcing}},
author = {Nguyen, An Thanh and Matthew Lease and Byron C.\ Wallace},
booktitle = {Proceedings of the 24th Annual ACM Intelligent User Interfaces (IUI) conference},
year = {2019},
pages = {575--579},
data = {http://github.com/thanhan/explainable-crowd-iui19},
url = {../papers/nguyen-iui19.pdf}
}
@inproceedings{Nguyen18-aaai,
title = {An Interpretable Joint Graphical Model for Fact-Checking from Crowds},
author = {Nguyen, An Thanh and Aditya Kharosekar and Matthew Lease and Byron C.\ Wallace},
booktitle = {Proceedings of the Thirty-Second AAAI Conference on Artificial Intelligence (AAAI-18)},
year = {2018},
url = {../papers/nguyen-aaai18.pdf},
source = {https://github.com/utir/fc-aaai18-py3},
slides = {https://www.slideshare.net/mattlease/believe-it-or-not-designing-a-humanai-partnership-for-mixedinitiative-factchecking/},
video = {https://youtu.be/TlY_Acr7gYs},
demo = {https://exfacto.herokuapp.com/},
pages = {1511--1518}
}
@inproceedings{Nguyen18-uist,
title = {Believe it or not: Designing a Human-AI Partnership for Mixed-Initiative Fact-Checking},
author = {Nguyen, An Thanh and Aditya Kharosekar and Saumyaa Krishnan and Siddhesh Krishnan and Elizabeth Tate and Byron C.\ Wallace and Matthew Lease},
booktitle = {Proceedings of the 31st ACM User Interface Software and Technology Symposium (UIST)},
year = {2018},
url = {../papers/nguyen-uist18.pdf},
slides = {https://www.slideshare.net/mattlease/believe-it-or-not-designing-a-humanai-partnership-for-mixedinitiative-factchecking/},
source = {https://github.com/thanhan/uist18},
video = {https://youtu.be/TlY_Acr7gYs},
demo = {https://exfacto.herokuapp.com/},
pages = {189-199},
note = {Interface sourcecode: https://github.com/utir/fcweb2-py3.}
}
@inproceedings{Nguyen17-acl,
author = {Nguyen, An Thanh and Junyi Jessy Li and Ani Nenkova and Byron C.\ Wallace and Matthew Lease},
title = {{Aggregating and Predicting Sequence Labels from Crowd Annotations}},
booktitle = {{Proceedings of the 55th annual meeting of the Association for Computational Linguistics (ACL)}},
year = {2017},
url = {../papers/nguyen-acl17.pdf},
source = {https://github.com/thanhan/seqcrowd-acl17},
data = {https://github.com/yinfeiy/PICO-data},
pages = {299--309}
}
@inproceedings{Nguyen16-hcomp,
author = {Nguyen, An Thanh and Matthew Halpern and Byron C.\ Wallace and Matthew Lease},
title = {{Probabilistic Modeling for Crowdsourcing Partially-Subjective Ratings}},
booktitle = {{Proceedings of the 4th AAAI Conference on Human Computation and Crowdsourcing (HCOMP)}},
year = {2016},
pages = {149--158},
source = {http://github.com/thanhan/subjective-crowd-hcomp16},
data = {http://github.com/Matthalp/mobile-cpu-user-sat-data.git},
blog = {https://humancomputation.com/blog/?p=9458},
url = {../papers/nguyen-hcomp16.pdf}
}
@inproceedings{Nguyen16-uai,
title = {A Correlated Worker Model for Grouped, Imbalanced and Multitask Data},
author = {Nguyen, An Thanh and Byron C.\ Wallace and Matthew Lease},
booktitle = {Proceedings of the 32nd International Conference on Uncertainty in Artificial Intelligence (UAI)},
year = {2016},
source = {https://github.com/thanhan/code-uai16},
url = {../papers/nguyen-uai16.pdf},
pages = {537--546}
}
@inproceedings{Nguyen15-hcomp,
author = {Nguyen, An Thanh and Byron C.\ Wallace and Matthew Lease},
title = {{Combining Crowd and Expert Labels using Decision Theoretic Active Learning}},
booktitle = {{Proceedings of the 3rd AAAI Conference on Human Computation (HCOMP)}},
year = {2015},
pages = {120--129},
url = {../papers/nguyen-hcomp15.pdf}
}
@incollection{Overgaard21,
author = {Christian Staal Bruun Overgaard and Anthony Dudo and Matthew Lease and Gina M. Masullo and Natalie Jomini Stroud and Scott R. Stroud and Samuel C. Woolley},
title = {Building connective democracy: Interdisciplinary solutions to the problem of polarisation},
booktitle = {The Routledge companion to Media Disinformation and Populism},
publisher = {Routledge},
year = 2021,
editor = {Tumber, Howard and Waisbord, Silvio},
volume = 1,
chapter = 53,
pages = {569--578},
note = {ISBN 9780367435769, SSRN 3831634},
techreport = {https://ssrn.com/abstract=3831634},
confurl = {https://www.routledge.com/The-Routledge-Companion-to-Media-Disinformation-and-Populism/Tumber-Waisbord/p/book/9780367435769}
}
@techreport{Petuchowski14-arxiv,
title = {TurKPF: TurKontrol as a Particle Filter},
author = {Ethan Petuchowski and Matthew Lease},
institution = {University of Texas at Austin},
month = {April},
day = {20},
year = {2014},
url = {http://arxiv.org/abs/1404.5078},
note = {arXiv:1404.5078},
source = {https://github.com/ethanp/crowdsourcing/tree/master/TurKPF}
}
@inproceedings{Rahman20-ictir,
title = {{Efficient Test Collection Construction via Active Learning}},
author = {Md Mustafizur Rahman and Mucahid Kutlu and Tamer Elsayed and Matthew Lease},
booktitle = {{Proceedings of The 6th ACM SIGIR International Conference on the Theory of Information Retrieval (ICTIR)}},
year = {2020},
pages = {177--184},
techreport = {http://arxiv.org/abs/1801.05605},
url = {../papers/rahman-ictir20.pdf}
}
@inproceedings{Rahman19-www,
title = {{Constructing Test Collections using Multi-armed Bandits and Active Learning}},
author = {Rahman, Md Mustafizur and Mucahid Kutlu and Matthew Lease},
booktitle = {{Proceedings of the Web Conference}},
year = {2019},
pages = {3158--3164},
url = {../papers/rahman-www19.pdf},
source = {https://github.com/mdmustafizurrahman/MAB_AL_TestCollection/}
}
@inproceedings{Roark06-icassp,
author = {Roark, B. and Yang Liu and Harper, M. and Stewart, R. and Lease, M. and Snover, M. and Shafran, I. and Dorr, B. and Hale, J. and Krasnyanskaya, A. and Yung, L.},
title = {Reranking for Sentence Boundary Detection in Conversational Speech},
booktitle = {Proceedings of IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP'06)},
year = 2006,
month = {May 14--19},
pages = {545--548},
url = {http://www.hlt.utdallas.edu/~yangl/papers/icassp2006.pdf}
}
@inproceedings{Roark06-lrec,
author = {Brian Roark and Mary Harper and Eugene Charniak and Bonnie Dorr and Mark Johnson and Jeremy G. Kahn and Yang Liu and Mari Ostendorf and John Hale and Anna Krasnyanskaya and Matthew Lease and Izhak Shafran and Matthew Snover and Robin Stewart and Lisa Yung},
title = {SParseval: Evaluation Metrics for Parsing Speech},
booktitle = {Fifth International Conference on Language Resources and Evaluation (LREC'06)},
year = 2006,
address = {Genoa, Italy},
url = {http://www.lrec-conf.org/proceedings/lrec2006/pdf/116_pdf.pdf}
}
@inproceedings{Ryu13-enrich,
author = {Hohyon Ryu and Matthew Lease},
title = {Generating Automatic Keywords for Conversational Speech ASR Transcripts},
booktitle = {1st ACM SIGIR Workshop on the Exploration, Navigation and Retrieval of Information in Cultural Heritage (ENRICH)},
year = 2013,
url = {../papers/ryu-enrich13.pdf},
note = {4 pages.},
confurl = {www.cultura-strep.eu/events/enrich-2013/}
}
@inproceedings{Ryu11-asist,
author = {Hohyon Ryu and Matthew Lease},
title = {Crowdworker Filtering with Support Vector Machine},
booktitle = {Proceedings of the 74th Annual Meeting of the American Society for Information Science and Technology (ASIS\&T)},
year = 2011,
url = {../papers/ryu-asist11.pdf},
note = {4 pages.}
}
@inproceedings{Ryu12-hypertext,
title = {Finding and Exploring Memes in Social Media},
author = {Hohyon Ryu and Matthew Lease and Nicholas Woodward},
booktitle = {{Proceedings of the 23rd ACM Conference on Hypertext and
Social Media}},
year = {2012},
pages = {295--304},
organization = {ACM},
source = {https://bitbucket.org/softbass/meme-extractor},
demo = {http://odyssey.ischool.utexas.edu/mb},
video = {http://www.youtube.com/watch?v=-Cvv5aVaVfI&feature=plcp},
url = {http://www.ischool.utexas.edu/~ml/papers/ryu-ht12.pdf},
note = {{\bf Winner}: 2012 \href{https://web.archive.org/web/20120623000849if_/https://www.umiacs.umd.edu/conferences/sbp2012/challenge.html}{Social Computing and Behavioral Modeling Prediction (SBP) Challenge}}
}
@inproceedings{Saha13,
author = {Ripon Saha and Matthew Lease and Sarfraz Khurshid and Dewayne Perry},
title = {{Improving Bug Localization using Structured Information Retrieval}},
booktitle = {{Proceedings of the 28th IEEE/ACM International Conference on Automated Software Engineering (ASE)}},
year = {2013},
confurl = {http://ase2013.org},
pages = {345--355},
data = {../data/ASE2013-BLUiR-Detailed-Results.zip},
url = {../papers/saha-ase13.pdf},
slides = {../papers/ASE-2024-slides.pdf},
note = {\textbf{ASE 2024 \href{https://ase-conferences.org/most-influential-papers}{Most Influential Paper Award}} (selection among papers from 2012-2014)}
}
@mastersthesis{Sheshadri-thesis14,
author = {Aashish Sheshadri},
title = {{A Collaborative Approach to IR Evaluation}},
school = {Department of Computer Science, University of Texas at Austin},
month = {May},
year = 2014,
note = {Co-Supervisors: Kristen Grauman and Matthew Lease.},
url = {https://repositories.lib.utexas.edu/bitstream/handle/2152/25910/SHESHADRI-THESIS-2014.pdf}
}
@inproceedings{Sheshadri13,
author = {Aashish Sheshadri and Matthew Lease},
title = {{SQUARE: A Benchmark for Research on Computing Crowd Consensus}},
booktitle = {{Proceedings of the 1st AAAI Conference on Human Computation (HCOMP)}},
year = {2013},
pages = {156--164},
url = {https://ojs.aaai.org/index.php/HCOMP/article/view/13088},
data = {http://ai.ischool.utexas.edu/square/},
slides = {https://utexas.box.com/v/lease-hcomp24-tot},
note = {\textbf{HCOMP 2024 Inaugural \href{https://www.humancomputation.com/past%20meetings.html#awards}{Test of Time and Impact Award}} (selection among papers from 2013-2014)}
}
@inproceedings{Sheshadri13-mediaeval,
author = {Aashish Sheshadri and Matthew Lease},
title = {{SQUARE: Benchmarking Crowd Consensus at MediaEval}},
booktitle = {{Proceedings of the MediaEval 2013 Multimedia Benchmark Workshop}},
year = {2013},
url = {http://ceur-ws.org/Vol-1043/mediaeval2013_submission_62.pdf},
confurl = {http://www.multimediaeval.org/mediaeval2013/crowd2013/},
data = {http://ai.ischool.utexas.edu/square/},
note = {2 pages. CEUR Workshop Proceedings Vol-1043, \url{http://ceur-ws.org/Vol-1043}.}
}
@techreport{Shira11,
author = {Elben Shira and Matthew Lease},
title = {Expert Search on Code Repositories},
institution = {Department of Computer Science, University of Texas at Austin},
number = {TR-11-42},
month = {December},
year = {2011},
url = {http://apps.cs.utexas.edu/tech_reports/reports/tr/TR-2066.pdf},
pages = {1--8}
}
@inproceedings{Shukla12-sigir,
author = {Shilpa Shukla and Matthew Lease and Ambuj Tewari},
title = {{Parallelizing ListNet Training using Spark}},
booktitle = {{Proceedings of the 35th international ACM SIGIR conference on Research and Development in Information Retrieval}},
year = {2012},
pages = {1127--1128},
source = {../data/shukla-sigir12-code.tar.gz},
url = {http://www.ischool.utexas.edu/~ml/papers/shukla-sigir12.pdf}
}
@techreport{singh2021hybrid,
title = {{A Hybrid 2-stage Neural Optimization for Pareto Front Extraction}},
author = {Gurpreet Singh and Soumyajit Gupta and Matthew Lease and Clint Dawson},
year = {2021},
number = {arXiv 2101.11684},
institution = {University of Texas at Austin},
url = {https://arxiv.org/pdf/2101.11684},
note = {January 27.}
}
@techreport{singh2020extracting,
title = {Extracting Optimal Solution Manifolds using Constrained Neural Optimization},
author = {Gurpreet Singh and Soumyajit Gupta and Matthew Lease},
year = {2020},
number = {arxiv 2009.06024},
institution = {University of Texas at Austin},
url = {https://arxiv.org/pdf/2009.06024},
note = {September 13.}
}
@inproceedings{Smucker14-cstrack,
author = {Mark D.\ Smucker and Gabriella Kazai and Matthew Lease},
title = {{Overview of the TREC 2013 Crowdsourcing Track}},
booktitle = {Proceedings of the 22nd NIST Text Retrieval Conference (TREC)},
url = {http://www.ischool.utexas.edu/~ml/papers/smucker-trec13.pdf},
confurl = {http://trec.nist.gov/pubs/trec22/t22.proceedings.html},
year = 2014,
note = {6 pages.}
}
@inproceedings{Smucker13-cstrack,
author = {Mark D.\ Smucker and Gabriella Kazai and Matthew Lease},
title = {{Overview of the TREC 2012 Crowdsourcing Track}},
booktitle = {Proceedings of the 21st NIST Text Retrieval Conference (TREC)},
year = 2013,
url = {http://trec.nist.gov/pubs/trec21/papers/CROWD12.overview.pdf},
confurl = {http://trec.nist.gov/pubs/trec21/t21.proceedings.html},
note = {12 pages.}
}
@inproceedings{Sun16-hcomp,
title = {{Crowdsourcing Information Extraction for Biomedical Systematic Reviews}},
author = {Yalin Sun and Pengxiang Cheng and Shengwei Wang and Hao Lyu and Matthew Lease and Iain Marshall and Byron C. Wallace},
booktitle = {{4th AAAI Conference on Human Computation and Crowdsourcing (HCOMP): Works-in-Progress Track}},
year = {2016},
url = {http://arxiv.org/abs/1609.01017},
note = {3 pages. arXiv:1609.01017}
}
@inproceedings{Suwaileh16,
author = {Reem Suwaileh and Mucahid Kutlu and Nihal Fathima and Tamer Elsayed and Matthew Lease},
title = {{ArabicWeb16: A New Crawl for Today's Arabic Web}},
booktitle = {Proceedings of the 39th international ACM SIGIR conference on Research and development in Information Retrieval},
pages = {673--676},
year = {2016},
url = {http://www.ischool.utexas.edu/~ml/papers/sigir16-arabicweb.pdf},
data = {https://sites.google.com/view/arabicweb16}
}
@inproceedings{Tang11-cir,
author = {Wei Tang and Matthew Lease},
title = {Semi-Supervised Consensus Labeling for Crowdsourcing},
booktitle = {{ACM SIGIR Workshop on Crowdsourcing for Information Retrieval (CIR)}},
year = {2011},
pages = {36--41},
url = {http://www.ischool.utexas.edu/~ml/papers/tang-cir11.pdf},
confurl = {https://sites.google.com/site/cir2011ws/program}
}
@inproceedings{Tian11-sigir,
author = {Aibo Tian and Matthew Lease},
title = {Active Learning to Maximize Accuracy vs. Effort in Interactive Information Retrieval},
booktitle = {{Proceedings of the 34th international ACM SIGIR conference on Research and Development in Information Retrieval}},
pages = {145--154},
year = {2011},
url = {http://www.ischool.utexas.edu/~ml/papers/tian-sigir11.pdf}
}
@inproceedings{Vakharia15-iconf,
title = {{Beyond Mechanical Turk: An Analysis of Paid Crowd Work Platforms}},
author = {Donna Vakharia and Matthew Lease},
booktitle = {{Proceedings of the iConference}},
techreport = {http://arxiv.org/pdf/1310.1672v1},
url = {http://www.ischool.utexas.edu/~ml/papers/donna-iconf15.pdf},
year = 2015,
note = {17 pages.}
}
@mastersthesis{Wolfson-thesis12,
author = {Stephen Wolfson},
title = {{Crowdsourcing and the Law}},
school = {School of Information, University of Texas at Austin},
month = {May},
year = 2012,
note = {Supervisor: Matthew Lease. Reader: James Howison.},
url = {http://repositories.lib.utexas.edu/handle/2152/ETD-UT-2012-05-5748}
}
@inproceedings{Wolfson11-asist,
author = {Stephen Wolfson and Matthew Lease},
title = {Look Before You Leap: Legal Pitfalls of Crowdsourcing},
booktitle = {Proceedings of the 74th Annual Meeting of the American Society for Information Science and Technology (ASIS\&T)},
url = {http://www.ischool.utexas.edu/~ml/papers/wolfson-asist11.pdf},
year = 2011,
confurl = {http://www.asis.org/asist2011},
techreport = {https://papers.ssrn.com/sol3/papers.cfm?abstract_id=4529384},
note = {10 pages.}
}
@techreport{Yang10-tr,
author = {Eunho Yang and Pradeep Ravikumar and Matthew Lease},
title = {{A new class of ranking functions for DCG-like evaluation metrics using conditional probability models}},
institution = {Department of Computer Science, University of Texas at Austin},
number = {AI14-02 (AI report)},
month = {October 29},
year = {2010},
url = {http://apps.cs.utexas.edu/tech_reports/reports/ai/AI-2175.pdf},
note = {8 pages.}
}
@phdthesis{zhang-dissertation19,
author = {Ye Zhang},
title = {{Neural NLP Models Under Low-supervision Scenarios}},
school = {Computer Science, University of Texas at Austin},
month = {May},
year = 2019,
url = {https://repositories.lib.utexas.edu/bitstream/handle/2152/75032/ZHANG-DISSERTATION-2019.pdf}
}
@article{zhang-irj18,
title = {{Neural Information Retrieval: At the End of the Early Years}},
author = {Kezban Dilek Onal and Ye Zhang and Ismail Sengor Altingovde and Md Mustafizur Rahman and Pinar Karagoz and Alexander Braylan and Brandon Dang and Heng-Lu Chang and Henna Kim and Quinten McNamara and Aaron Angert and Edward Banner and Vivek Khetan
and Tyler McDonnell and An Thanh Nguyen and Dan Xu and Byron C.\ Wallace and Maarten de Rijke and Matthew Lease},
journal = {Information Retrieval},
year = {2018},
volume = 21,
number = {2-3},
pages = {111-182},
publisher = {Springer},
issn = {1573-7659},
doi = {10.1007/s10791-017-9321-y},
techreport = {https://arxiv.org/abs/1611.06792},
slides = {http://www.slideshare.net/mattlease/deep-learning-for-information-retrieval-models-progress-opportunities},
url = {../papers/zhang-irj18.pdf}
}
@techreport{zhang-arxiv16,
author = {Ye Zhang and Md Mustafizur Rahman and Alex Braylan and Brandon Dang and Heng-Lu Chang and Henna Kim and Quinten McNamara and Aaron Angert and Edward Banner and Vivek Khetan and Tyler McDonnell and An Thanh Nguyen and Dan Xu and Byron C. Wallace and Matthew Lease},
title = {{Neural Information Retrieval: A Literature Review}},
institution = {University of Texas at Austin},
year = 2016,
month = {November},
url = {https://arxiv.org/abs/1611.06792},
slides = {http://www.slideshare.net/mattlease/deep-learning-for-information-retrieval-models-progress-opportunities},
note = {ArXiv 1611.06792.}
}
@inproceedings{zhang-acl17-weight,
author = {Ye Zhang and Matthew Lease and Byron C.\ Wallace},
title = {{Exploiting Domain Knowledge via Grouped Weight Sharing with Application to Text Categorization}},
booktitle = {{Proceedings of the 55th annual meeting of the Association for Computational Linguistics (ACL)}},
year = {2017},
url = {../papers/zhang-acl17.pdf},
pages = {155--160},
techreport = {https://arxiv.org/abs/1702.02535}
}
@inproceedings{Zhang-aaai17,
author = {Ye Zhang and Matthew Lease and Byron C.\ Wallace},
title = {Active Discriminative Text Representation Learning},
booktitle = {{Proceedings of the Thirty-First AAAI Conference on Artificial Intelligence (AAAI)}},
year = {2017},
url = {http://aaai.org/ocs/index.php/AAAI/AAAI17/paper/view/14174/14265},
pages = {3386-3392},
note = {Also accepted for encore presentation at the 2nd Workshop on Representation Learning for NLP (RepL4NLP) at the 55th Annual Meeting of the Association for Computational Linguistics (ACL).},
confurl = {https://sites.google.com/site/repl4nlp2017/}
}
@inproceedings{Zhang-sigir14,
author = {Yinglong Zhang and Jin Zhang and Matthew Lease and Jacek Gwizdka},
title = {Multidimensional Relevance Modeling via Psychometrics and Crowdsourcing},
booktitle = {{Proceedings of the 37th international ACM SIGIR conference on Research and Development in Information Retrieval}},
pages = {435--444},
year = {2014},
data = {../data/zhang-sigir14-data.zip},
url = {../papers/zhang-sigir14.pdf}
}
@inproceedings{Zheng-apccm17,
author = {Xi Zheng and Akanksha Bansal and Matthew Lease},
title = {Bullseye: Structured Passage Retrieval and Document Highlighting for Scholarly Search},
booktitle = {{The Thirteenth Asia-Pacific Conference on Conceptual Modelling (APCCM), held as part of the Australasian Computer Science Week (ACSW) Multiconference}},
year = {2017},
isbn = {978-1-4503-4768-6},
location = {Geelong, Australia},
articleno = {32},
numpages = {4},
url = {http://doi.acm.org/10.1145/3014812.3014846},
doi = {10.1145/3014812.3014846},
acmid = {3014846},
url = {../papers/Zheng-APCCM17.pdf},
techreport = {https://arxiv.org/abs/1406.1875},
confurl = {http://www.dsl.uow.edu.au/main/apccm2017/},
note = {4 pages.}
}
@mastersthesis{Zhou-thesis15,
author = {Haofeng Zhou},
title = {{Crowdsourcing Construction of Information Retrieval Test Collections for Conversational Speech}},
school = {School of Information, University of Texas at Austin},
month = {May},
year = 2015,
note = {Reader: Byron C.\ Wallace},
url = {https://repositories.lib.utexas.edu/bitstream/handle/2152/31916/ZHOU-THESIS-2015.pdf}
}
@inproceedings{Zhou13-sslud,
author = {Haofeng Zhou and Dennis Baskov and Matthew Lease},
title = {Crowdsourcing Transcription Beyond Mechanical Turk},
booktitle = {AAAI HCOMP Workshop on Scaling Speech, Language Understanding and Dialogue through Crowdsourcing (SSLUD)},
year = {2013},
confurl = {http://faculty.washington.edu/mtjalve/HCOMP2013.Workshop.html},
url = {../papers/zhou-sslud13.pdf},
note = {8 pages}
}
@inproceedings{Zhou:2011:MOO:1940761.1940842,
author = {Zhou, Yongyi and Broussard, Ramona and Lease, Matthew},
title = {Mobile options for online public access catalogs},
booktitle = {{Proceedings of the iConference}},
year = {2011},
isbn = {978-1-4503-0121-3},
location = {Seattle, Washington},
acmid = {1940842},
publisher = {{ACM}},
pages = {598--605},
numpages = {8},
video = {../papers/zhou-brossard-mut.avi},
url = {http://www.ischool.utexas.edu/~ml/papers/zhou-iconf10.pdf},
confurl = {http://www.ischools.org/iConference11/2011index}
}
@conference{Lamarca02,
title = {{PlantCare: An Investigation in Practical Ubiquitous Systems}},
booktitle = {Proceedings of the 4th International Conference on Ubiquitous Computing (UBICOMP)},
author = {LaMarca, A. and Brunette, W. and Koizumi, D. and Lease, M. and Sigurdsson, S.B. and Sikorski, K. and Fox, D. and Borriello, G.},
series = {LECTURE NOTES IN COMPUTER SCIENCE},
volume = 2498,
pages = {316--332},
year = {2002},
publisher = {Springer},
url = {http://www.intel-research.net/Publications/Seattle/072320021545_76.pdf}
}
@inproceedings{Sigurdsson02,
author = {Anthony LaMarca and Waylon Brunette and David Koizumi and Matthew Lease and Stefan B. Sigurdsson and Kevin Sikorski and Dieter Fox and Gaetano Borriello},
booktitle = {Pervasive '02: Proceedings of the First International Conference on Pervasive Computing},
title = {Making Sensor Networks Practical with Robots},
year = {2002},
pages = {152--166},
publisher = {Springer-Verlag},
series = {LECTURE NOTES IN COMPUTER SCIENCE},
volume = 2414,
url = {http://link.springer.com/content/pdf/10.1007%2F3-540-45866-2_13.pdf}
}
@conference{Kalet99,
title = {{Anatomical information in radiation treatment planning}},
author = {Kalet, I.J. and Wu, J. and Lease, M. and Austin-Seymour, M.M. and Brinkley, J.F. and Rosse, C.},
booktitle = {Proceedings of the American Medical Informatics Association (AMIA) Fall Symposium},
year = {1999},
url = {http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2232501/pdf/procamiasymp00004-0328.pdf}
}
@conference{Kalet98,
title = {{Radiation Therapy Planning: an Uncommon Application of Lisp}},
author = {Kalet, I.J. and Giansiracusa, R.S. and Wilcox, C. and Lease, M.},
booktitle = {Proceedings of the Conference on the 40th Anniversary of Lisp},
editor = {R. Gabriel},
year = {1998},
url = {../papers/lisp40yrs.pdf}
}
This file was generated by bibtex2html 1.99.