LLM Hallucination Testing: A QA Engineer’s Guide to Detecting AI Fabrications

Hallucinations are the #1 risk in LLM-powered features. QA engineers need systematic methods to detect, measure, and prevent them.

🤖 Learning AI-powered testing? Go hands-on with LLM, RAG, and AI-agent testing in the AI-Powered Testing Mastery course at The Testing Academy.

Types of Hallucinations

Type	Example	Detection Method
Factual	Invents statistics or dates	Fact verification against source
Citation	Cites non-existent papers	Citation validation
Logical	Contradicts itself mid-response	Consistency checking
Contextual	Answers unrelated to question	Relevance scoring
Fabricated entity	Invents people, companies	Entity verification

Hallucination Detection Framework

class HallucinationDetector:
    def detect_factual(self, response, ground_truth):
        claims = self.extract_claims(response)
        verified = [c for c in claims if self.verify(c, ground_truth)]
        return {
            "total_claims": len(claims),
            "verified": len(verified),
            "hallucinated": len(claims) - len(verified),
            "rate": 1 - len(verified) / max(len(claims), 1)
        }

    def detect_self_contradiction(self, response):
        sentences = self.split_sentences(response)
        contradictions = []
        for i, s1 in enumerate(sentences):
            for s2 in sentences[i+1:]:
                if self.contradicts(s1, s2):
                    contradictions.append((s1, s2))
        return contradictions

    def detect_fabricated_entities(self, response):
        entities = self.extract_entities(response)
        fabricated = [e for e in entities if not self.entity_exists(e)]
        return fabricated

Grounding Score

def calculate_grounding_score(response, source_documents):
    # For RAG: how much of response is grounded in sources
    sentences = split_sentences(response)
    grounded = 0
    for s in sentences:
        for doc in source_documents:
            if semantic_similarity(s, doc) > 0.8:
                grounded += 1
                break
    return grounded / len(sentences)

# Target: grounding score > 0.95 for production

Hallucination Test Suite

@pytest.fixture
def detector():
    return HallucinationDetector()

def test_no_factual_hallucination(detector, llm):
    response = llm.generate("List 5 Playwright features")
    result = detector.detect_factual(response, PLAYWRIGHT_FACTS)
    assert result["rate"] < 0.05

def test_no_fabricated_citations(detector, llm):
    response = llm.generate("Cite sources for test automation benefits")
    fabricated = detector.detect_fabricated_entities(response)
    assert len(fabricated) == 0

def test_self_consistency(detector, llm):
    response = llm.generate("Compare Selenium and Playwright")
    contradictions = detector.detect_self_contradiction(response)
    assert len(contradictions) == 0