@inproceedings{lee2026knowsim,title={KNOWSIM: Evaluating Information Calibration in LLM Assistants with User Simulators that Learn},author={Lee, Yoonjoo and Jin, Hyoungwook and Kim, Tae Soo and Zhang, Shaoyang and Laban, Philippe and Liao, Q. Vera},booktitle={Proceedings of the Conference on Empirical Methods in Natural Language Processing (Main Conference)},year={2026},}
@inproceedings{jeong2026assistantbias,title={Investigating Assistant Bias in LLM User Simulators Using a Role Vector},author={Jeong, Daeheon and Lee, Yoonjoo and Choi, Eugene and {van der Ben}, Sinie and Kim, Juho},booktitle={Findings of the Conference on Empirical Methods in Natural Language Processing},year={2026},}
@inproceedings{adar2026reviving,title={Revibing Code from Papers: Reimplementing HCI Artifacts},author={Adar, Eytan and Lee, Yoonjoo and Lei, Ning-Er and Liao, Q. Vera and Peng, Weirui},booktitle={Proceedings of the ACM Symposium on User Interface Software and Technology},year={2026},}
To handle ambiguous and open-ended requests, Large Language Models (LLMs) are increasingly trained to interact with users to surface intents they have not yet expressed (e.g., ask clarification questions). However, users are often ambiguous because they have not yet formed their intents: they must observe and explore outcomes to discover what they want. Simply asking "what kind of tone do you want?" fails when users themselves do not know. We introduce DiscoverLLM, a novel and generalizable framework that trains LLMs to help users form and discover their intents. Central to our approach is a novel user simulator that models cognitive state with a hierarchy of intents that progressively concretize as the model surfaces relevant options – where the degree of concretization serves as a reward signal that models can be trained to optimize. Resulting models learn to collaborate with users by adaptively diverging (i.e., explore options) when intents are unclear, and converging (i.e., refine and implement) when intents concretize. Across proposed interactive benchmarks in creative writing, technical writing, and SVG drawing, DiscoverLLM achieves over 10% higher task performance while reducing conversation length by up to 40%. In a user study with 75 human participants, DiscoverLLM improved conversation satisfaction and efficiency compared to baselines.
@inproceedings{kim2026discoverllm,title={DiscoverLLM: From Executing Intents to Discovering Them},author={Kim, Tae Soo and Lee, Yoonjoo and Yu, Jaesang and Chung, John Joon Young and Kim, Juho},booktitle={Proceedings of the International Conference on Machine Learning},year={2026},}
Practitioners increasingly rely on Large Language Models (LLMs) to evaluate generative AI outputs through "LLM-as-a-Judge" approaches. However, these methods produce holistic scores that obscure which specific elements influenced the assessments. We propose functional fragmentation, a method that dissects each output into key fragments and interprets the rhetoric functions that each fragment serves relative to evaluation criteria – surfacing the elements of interest and revealing how they fulfill or hinder user goals. We instantiate this approach in Evalet, an interactive system that visualizes fragment-level functions across many outputs to support inspection, rating, and comparison of evaluations. A user study (N=10) found that, while practitioners struggled to validate holistic scores, our approach helped them identify 48% more evaluation misalignments. This helped them calibrate trust in LLM evaluations and rely on them to find more actionable issues in model outputs. Our work shifts LLM evaluation from quantitative scores toward qualitative, fine-grained analysis of model behavior.
@inproceedings{kim2026evalet,title={Evalet: Evaluating Large Language Models through Functional Fragmentation},author={Kim, Tae Soo and Lee, Heechan and Lee, Yoonjoo and Seering, Joseph and Kim, Juho},booktitle={Proceedings of the CHI Conference on Human Factors in Computing Systems},year={2026},}
Patent examination remains an ongoing challenge in the NLP literature even after the advent of large language models (LLMs), as it requires an extensive yet nuanced human judgment on whether a submitted claim meets the statutory standards of novelty and non-obviousness against previously granted claims – prior art – in expert domains. Previous NLP studies have approached this challenge as a prediction task (e.g., forecasting grant outcomes) with high-level proxies such as similarity metrics or classifiers trained on historical labels. However, this approach often overlooks the step-by-step evaluations that examiners must make with profound information, including rationales for the decisions provided in office actions documents, which also makes it harder to measure the current state of techniques in patent review processes. To fill this gap, we construct PANORAMA, a dataset of 8,143 U.S. patent examination records that preserves the full decision trails, including original applications, all cited references, Non-Final Rejections, and Notices of Allowance. Also, PANORAMA decomposes the trails into sequential benchmarks that emulate patent professionals’ patent review processes and allow researchers to examine large language models’ capabilities at each step of them. Our findings indicate that, although LLMs are relatively effective at retrieving relevant prior art and pinpointing the pertinent paragraphs, they struggle to assess the novelty and non-obviousness of patent claims. We discuss these results and argue that advancing NLP, including LLMs, in the patent domain requires a deeper understanding of real-world patent examination. Our dataset is openly available at https://huggingface.co/datasets/LG-AI-Research/PANORAMA.
@inproceedings{lim2025panorama,title={PANORAMA: A Dataset and Benchmarks Capturing Decision Trails and Rationales in Patent Examination},author={Lim, Hyunseung and Nam, Sooyohn and Na, Sungmin and Cho, Ji Yong and Yang, June Yong and Shin, Hyungyu and Lee, Yoonjoo and Kim, Juho and Lee, Moontae and Hong, Hwajung},booktitle={Advances in Neural Information Processing Systems},year={2025},}
Personalization of Large Language Models (LLMs) often assumes users hold static preferences that reflect globally in all tasks. In reality, humans hold dynamic preferences that change depending on the context. As users interact with an LLM in various contexts, they naturally reveal their contextual preferences, which a model must infer and apply in future contexts to ensure alignment. To assess this, we introduce CUPID, a benchmark of 756 human-curated interaction session histories between users and LLM-based chat assistants. In each interaction session, the user provides a request in a specific context and expresses their preference through multi-turn feedback. Given a new user request and prior interaction sessions, our benchmark assesses whether LLMs can infer the preference relevant to this request and generate a response that satisfies this preference. With CUPID, we evaluated 10 open and proprietary LLMs, revealing that state-of-the-art LLMs struggle to infer preferences from multi-turn interactions and fail to discern what previous context is relevant to a new request – under 50% precision and 65% recall. Our work highlights the need to advance LLM capabilities for more contextually personalized interactions and proposes CUPID as a resource to drive these improvements.
@inproceedings{kim2025cupid,title={CUPID: Evaluating Personalized and Contextualized Alignment of LLMs from Interactions},author={Kim, Tae Soo and Lee, Yoonjoo and Park, Yoonah and Kim, Jiho and Kim, Young-Ho and Kim, Juho},booktitle={Conference on Language Modeling},year={2025},}
Peer review underpins scientific progress, but it is increasingly strained by reviewer shortages and growing workloads. Large Language Models (LLMs) can automatically draft reviews now, but determining whether LLM-generated reviews are trustworthy requires systematic evaluation. Researchers have evaluated LLM reviews at either surface-level (e.g., BLEU and ROUGE) or content-level (e.g., specificity and factual accuracy). Yet it remains uncertain whether LLM-generated reviews attend to the same critical facets that human experts weigh – the strengths and weaknesses that ultimately drive an accept-or-reject decision. We introduce a focus-level evaluation framework that operationalizes the focus as a normalized distribution of attention across predefined facets in paper reviews. Based on the framework, we developed an automatic focus-level evaluation pipeline based on two sets of facets: target (e.g., problem, method, and experiment) and aspect (e.g., validity, clarity, and novelty), leveraging 676 paper reviews (https://figshare.com/s/d5adf26c802527dd0f62) from OpenReview that consists of 3,657 strengths and weaknesses identified from human experts. The comparison of focus distributions between LLMs and human experts showed that the off-the-shelf LLMs consistently have a more biased focus towards examining technical validity while significantly overlooking novelty assessment when criticizing papers.
@inproceedings{shin2025blindspots,title={Mind the Blind Spots: A Focus-Level Evaluation Framework for LLM Reviews},author={Shin, Hyungyu and Tang, Jingyu and Lee, Yoonjoo and Kim, Nayoung and Lim, Hyunseung and Cho, Ji Yong and Hong, Hwajung and Lee, Moontae and Kim, Juho},booktitle={Proceedings of the Conference on Empirical Methods in Natural Language Processing (Main Conference)},year={2025},}
The BiGGen Bench: A Principled Benchmark for Fine-grained Evaluation of Language Models with Language Models
Seungone Kim, Juyoung Suk, Ji Yong Cho, Shayne Longpre, Chaeeun Kim, Dongkeun Yoon, Guijin Son, Yejin Cho, Sheikh Shafayat, Jinheon Baek, Sue Hyun Park, Hyeonbin Hwang, Jinkyung Jo, Hyowon Cho, Haebin Shin, Seongyun Lee, Hanseok Oh, Noah Lee, Namgyu Ho, Se June Joo, Miyoung Ko, Yoonjoo Lee, Hyungjoo Chae, Jamin Shin, Joel Jang, Seonghyeon Ye, Bill Yuchen Lin, Sean Welleck, Graham Neubig, Moontae Lee, Kyungjae Lee, and Minjoon Seo
In Proceedings of the Conference of the North American Chapter of the Association for Computational Linguistics, 2025
As language models (LMs) become capable of handling a wide range of tasks, their evaluation is becoming as challenging as their development. Most generation benchmarks currently assess LMs using abstract evaluation criteria like helpfulness and harmlessness, which often lack the flexibility and granularity of human assessment. Additionally, these benchmarks tend to focus disproportionately on specific capabilities such as instruction following, leading to coverage bias. To overcome these limitations, we introduce the BiGGen Bench, a principled generation benchmark designed to thoroughly evaluate nine distinct capabilities of LMs across 77 diverse tasks. A key feature of the BiGGen Bench is its use of instance-specific evaluation criteria, closely mirroring the nuanced discernment of human evaluation. We apply this benchmark to assess 103 frontier LMs using five evaluator LMs. Our code, data, and evaluation results are all publicly available at https://github.com/prometheus-eval/prometheus-eval/tree/main/BiGGen-Bench.
@inproceedings{kim2025biggen,title={The BiGGen Bench: A Principled Benchmark for Fine-grained Evaluation of Language Models with Language Models},author={Kim, Seungone and Suk, Juyoung and Cho, Ji Yong and Longpre, Shayne and Kim, Chaeeun and Yoon, Dongkeun and Son, Guijin and Cho, Yejin and Shafayat, Sheikh and Baek, Jinheon and Park, Sue Hyun and Hwang, Hyeonbin and Jo, Jinkyung and Cho, Hyowon and Shin, Haebin and Lee, Seongyun and Oh, Hanseok and Lee, Noah and Ho, Namgyu and Joo, Se June and Ko, Miyoung and Lee, Yoonjoo and Chae, Hyungjoo and Shin, Jamin and Jang, Joel and Ye, Seonghyeon and Lin, Bill Yuchen and Welleck, Sean and Neubig, Graham and Lee, Moontae and Lee, Kyungjae and Seo, Minjoon},booktitle={Proceedings of the Conference of the North American Chapter of the Association for Computational Linguistics},year={2025},}
When conducting literature reviews, scientists often create literature review tables - tables whose rows are publications and whose columns constitute a schema, a set of aspects used to compare and contrast the papers. Can we automatically generate these tables using language models (LMs)? In this work, we introduce a framework that leverages LMs to perform this task by decomposing it into separate schema and value generation steps. To enable experimentation, we address two main challenges: First, we overcome a lack of high-quality datasets to benchmark table generation by curating and releasing arxivDIGESTables, a new dataset of 2,228 literature review tables extracted from ArXiv papers that synthesize a total of 7,542 research papers. Second, to support scalable evaluation of model generations against human-authored reference tables, we develop DecontextEval, an automatic evaluation method that aligns elements of tables with the same underlying aspects despite differing surface forms. Given these tools, we evaluate LMs’ abilities to reconstruct reference tables, finding this task benefits from additional context to ground the generation (e.g. table captions, in-text references). Finally, through a human evaluation study we find that even when LMs fail to fully reconstruct a reference table, their generated novel aspects can still be useful.
@inproceedings{newman2024arxivdigestables,title={ArxivDIGESTables: Synthesizing Scientific Literature into Tables using Language Models},author={Newman, Benjamin and Lee, Yoonjoo and Naik, Aakanksha and Siangliulue, Pao and Fok, Raymond and Kim, Juho and Weld, Daniel S. and Chang, Joseph Chee and Lo, Kyle},booktitle={Proceedings of the Conference on Empirical Methods in Natural Language Processing (Main Conference)},year={2024},}
As Large Language Models (LLMs) are nondeterministic, the same input can generate different outputs, some of which may be incorrect or hallucinated. If run again, the LLM may correct itself and produce the correct answer. Unfortunately, most LLM-powered systems resort to single results which, correct or not, users accept. Having the LLM produce multiple outputs may help identify disagreements or alternatives. However, it is not obvious how the user will interpret conflicts or inconsistencies. To this end, we investigate how users perceive the AI model and comprehend the generated information when they receive multiple, potentially inconsistent, outputs. Through a preliminary study, we identified five types of output inconsistencies. Based on these categories, we conducted a study (N=252) in which participants were given one or more LLM-generated passages to an information-seeking question. We found that inconsistency within multiple LLM-generated outputs lowered the participants’ perceived AI capacity, while also increasing their comprehension of the given information. Specifically, we observed that this positive effect of inconsistencies was most significant for participants who read two passages, compared to those who read three. Based on these findings, we present design implications that, instead of regarding LLM output inconsistencies as a drawback, we can reveal the potential inconsistencies to transparently indicate the limitations of these models and promote critical LLM usage.
@inproceedings{lee2024onevsmany,title={One vs. Many: Comprehending Accurate Information from Multiple Erroneous and Inconsistent AI Generations},author={Lee, Yoonjoo and Son, Kihoon and Kim, Tae Soo and Kim, Jisu and Chung, John Joon Young and Adar, Eytan and Kim, Juho},booktitle={Proceedings of the ACM Conference on Fairness, Accountability, and Transparency},year={2024},}
With the rapid growth of scholarly archives, researchers subscribe to "paper alert" systems that periodically provide them with recommendations of recently published papers that are similar to previously collected papers. However, researchers sometimes struggle to make sense of nuanced connections between recommended papers and their own research context, as existing systems only present paper titles and abstracts. To help researchers spot these connections, we present PaperWeaver, an enriched paper alerts system that provides contextualized text descriptions of recommended papers based on user-collected papers. PaperWeaver employs a computational method based on Large Language Models (LLMs) to infer users’ research interests from their collected papers, extract context-specific aspects of papers, and compare recommended and collected papers on these aspects. Our user study (N=15) showed that participants using PaperWeaver were able to better understand the relevance of recommended papers and triage them more confidently when compared to a baseline that presented the related work sections from recommended papers.
@inproceedings{lee2024paperweaver,title={PaperWeaver: Enriching Topical Paper Alerts by Contextualizing Recommended Papers with User-collected Papers},author={Lee, Yoonjoo and Kang, Hyeonsu B. and Latzke, Matt and Kim, Juho and Bragg, Jonathan and Chang, Joseph Chee and Siangliulue, Pao},booktitle={Proceedings of the CHI Conference on Human Factors in Computing Systems},year={2024},}
By simply composing prompts, developers can prototype novel generative applications with Large Language Models (LLMs). To refine prototypes into products, however, developers must iteratively revise prompts by evaluating outputs to diagnose weaknesses. Formative interviews (N=8) revealed that developers invest significant effort in manually evaluating outputs as they assess context-specific and subjective criteria. We present EvalLM, an interactive system for iteratively refining prompts by evaluating multiple outputs on user-defined criteria. By describing criteria in natural language, users can employ the system’s LLM-based evaluator to get an overview of where prompts excel or fail, and improve these based on the evaluator’s feedback. A comparative study (N=12) showed that EvalLM, when compared to manual evaluation, helped participants compose more diverse criteria, examine twice as many outputs, and reach satisfactory prompts with 59% fewer revisions. Beyond prompts, our work can be extended to augment model evaluation and alignment in specific application contexts.
@inproceedings{kim2024evallm,title={EvalLM: Interactive Evaluation of Large Language Model Prompts on User-Defined Criteria},author={Kim, Tae Soo and Lee, Yoonjoo and Shin, Jamin and Kim, Young-Ho and Kim, Juho},booktitle={Proceedings of the CHI Conference on Human Factors in Computing Systems},year={2024},}
The lengthy monologue-style online lectures cause learners to lose engagement easily. Designing lectures in a "vicarious dialogue" format can foster learners’ cognitive activities more than monologue-style. However, designing online lectures in a dialogue style catered to the diverse needs of learners is laborious for instructors. We conducted a design workshop with eight educational experts and seven instructors to present key guidelines and the potential use of large language models (LLM) to transform a monologue lecture script into pedagogically meaningful dialogue. Applying these design guidelines, we created VIVID which allows instructors to collaborate with LLMs to design, evaluate, and modify pedagogical dialogues. In a within-subjects study with instructors (N=12), we show that VIVID helped instructors select and revise dialogues efficiently, thereby supporting the authoring of quality dialogues. Our findings demonstrate the potential of LLMs to assist instructors with creating high-quality educational dialogues across various learning stages.
@inproceedings{choi2024vivid,title={VIVID: Human-AI Collaborative Authoring of Vicarious Dialogues from Lecture Videos},author={Choi, Seulgi and Lee, Hyewon and Lee, Yoonjoo and Kim, Juho},booktitle={Proceedings of the CHI Conference on Human Factors in Computing Systems},year={2024},}
CSAM
Forecasting realized volatility using data normalization and recurrent neural network
Yoonjoo Lee, Dong Wan Shin, and Ji Eun Choi
Communications for Statistical Applications and Methods, 2024
@article{lee2024csam,title={Forecasting realized volatility using data normalization and recurrent neural network},author={Lee, Yoonjoo and Shin, Dong Wan and Choi, Ji Eun},journal={Communications for Statistical Applications and Methods},volume={31},number={1},pages={1--23},year={2024},}
@inproceedings{lee2023qasa,title={QASA: Answering Advanced Questions on Scientific Articles},author={Lee, Yoonjoo and Lee, Kyungjae and Park, Sunghyun and Hwang, Dasol and Kim, Jaehyeon and Lee, Hong-in and Lee, Moontae},booktitle={Proceedings of the International Conference on Machine Learning},year={2023},}
@inproceedings{kim2023cells,title={Cells, Generators, and Lenses: Design Framework for Object-Oriented Interaction with Large Language Models},author={Kim, Tae Soo and Lee, Yoonjoo and Chang, Minsuk and Kim, Juho},booktitle={Proceedings of the ACM Symposium on User Interface Software and Technology},year={2023},}
@inproceedings{lee2023dapie,title={DAPIE: Interactive Step-by-Step Explanatory Dialogues to Answer Children's Why and How Questions},author={Lee, Yoonjoo and Kim, Tae Soo and Kim, Sungdong and Yun, Yohan and Kim, Juho},booktitle={Proceedings of the CHI Conference on Human Factors in Computing Systems},year={2023},}
@inproceedings{lee2022promptiverse,title={Promptiverse: Scalable Generation of Scaffolding Prompts Through Human-AI Hybrid Knowledge Graph Annotation},author={Lee, Yoonjoo and Chung, John Joon Young and Kim, Tae Soo and Song, Jean Y. and Kim, Juho},booktitle={Proceedings of the CHI Conference on Human Factors in Computing Systems},year={2022},}
@inproceedings{lee2021studywithme,title={Personalizing Ambience and Illusionary Presence: How People Use ``Study with Me'' Videos to Create Effective Studying Environments},author={Lee, Yoonjoo and Chung, John Joon Young and Song, Jean Y. and Chang, Minsuk and Kim, Juho},booktitle={Proceedings of the CHI Conference on Human Factors in Computing Systems},year={2021},}
2019
ICTC
A Machine Learning Approach that meets Axiomatic Properties in Probabilistic Analysis of LTE Spectral Efficiency
Yoonjoo Lee, Yunbae Kim, and Seungken Park
In International Conference on Information and Communication Technology Convergence (ICTC), Oct 2019
@inproceedings{lee2019ictc,title={A Machine Learning Approach that meets Axiomatic Properties in Probabilistic Analysis of LTE Spectral Efficiency},author={Lee, Yoonjoo and Kim, Yunbae and Park, Seungken},booktitle={International Conference on Information and Communication Technology Convergence (ICTC)},year={2019},month=oct,}
IEEE
Probabilistic Analysis of Spectral Efficiency for LTE based on PDCCH Measurement Data
Yoonjoo Lee, Yunbae Kim, Yeongyu Park, and Seungken Park
@article{lee2019commlett,title={Probabilistic Analysis of Spectral Efficiency for LTE based on PDCCH Measurement Data},author={Lee, Yoonjoo and Kim, Yunbae and Park, Yeongyu and Park, Seungken},journal={IEEE Communications Letters},volume={23},number={9},year={2019},month=sep,}
2018
KICS
A Study on the Distribution Analysis of LTE Resource Block Usage from TSME Measurement Data
Yoonjoo Lee, Yunbae Kim, and Seungken Park
In Korean Institute of Communications and Information Sciences (KICS) Summer Conference, Jun 2018