kenschae/W1---Opportunity-Intelligence
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

W1-021 calibrate owner qualification profile

Browse Source
This commit is contained in:
kenschae
2026-09-17 15:04:24 -05:00
parent 63a24760e1
commit 79167dadfb
8 changed files with 927 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2
View File
@@ -80,3 +80,5 @@ source conflicts and human decisions. HTTP availability is not confirmed hiring
status; candidate outputs remain unreviewed until Ken approves promotion.
W1-019 adds `python w1.py practical --help` for saved-evidence practical eligibility. See [profile editing, rules and replay](docs/PRACTICAL-QUALIFICATION.md).
[W1-021 owner profile calibration and offline comparison](docs/OWNER-PROFILE-CALIBRATION.md).
data/README.md
+4
View File
@@ -8,3 +8,7 @@ The immutable test copy is under `tests/fixtures/`; historical pre-refactor byte
are retained under `runs/2026-09-17/w1-017/before/`.
`qualification-profile.json` contains owner-editable W1-019 criteria and explicitly unknown personal facts. See [editing and decision rules](../docs/PRACTICAL-QUALIFICATION.md).
The profile now uses schema version 2 with owner-supplied career evidence.
See [evidence maintenance](../docs/OWNER-PROFILE-CALIBRATION.md) before editing
statuses, qualification thresholds or aliases. Unknown personal facts remain unknown.
data/qualification-profile.json
+270 -4
View File
@@ -1,5 +1,5 @@
{
"schema_version": 1,
{
"schema_version": 2,
"owner": "Ken Schaefer",
"opportunity_categories": [
"full_time_employment",
@@ -48,13 +48,279 @@
"security_clearance": "unknown",
"willing_to_relocate": "unknown",
"willing_to_travel": "unknown",
"years_of_experience": "unknown",
"years_of_experience": 30,
"certifications": "unknown",
"specialist_skills": "unknown"
},
"notes": [
"Geographic scope does not establish actual residence or relocation willingness.",
"Specialist mismatch means a core-function preference mismatch, not proven lack of skills.",
"Owner can set a specialist family to consider, or add approved onsite states. Unknown requirements still need review."
"Owner can set a specialist family to consider, or add approved onsite states. Unknown requirements still need review.",
"Career evidence status is established, unknown, or excluded (explicit owner preference). Never infer excluded from unknown.",
"Aliases identify supported aspects, not every clause in a compound requirement. Familiarity never proves specialist depth.",
"years_supported is a conservative qualification threshold, not an invented exact tenure. Tenure subjects require full normalized correspondence.",
"Eight years architecture leadership is accepted explicitly by W1-021; do not derive specialist tenure from total IT years.",
"Past client names are context only and never automatically satisfy an industry credential or specialization."
],
"career_evidence": [
{
"id": "it_career",
"status": "established",
"statement": "More than 30 years of IT experience.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"IT experience"
],
"years_supported": 30,
"tenure_subjects": [
"IT",
"information technology",
"IT experience"
]
},
{
"id": "architecture",
"status": "established",
"statement": "More than 15 years of architecture experience; significant enterprise and solution architecture experience.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"enterprise architecture",
"solution architecture",
"architecture experience"
],
"years_supported": 15,
"tenure_subjects": [
"architecture",
"architecture experience",
"enterprise architecture",
"solution architecture",
"IT leadership / architecture",
"IT leadership / enterprise architecture / solution architecture",
"IT leadership, enterprise architecture, or solution architecture experience"
]
},
{
"id": "architecture_leadership",
"status": "established",
"statement": "IT leadership and technology-direction responsibilities; cross-functional technical leadership. W1-021 explicitly accepts the 8+ years architecture-leadership requirement.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"architecture leadership",
"technology direction",
"technical leadership",
"IT leadership"
],
"years_supported": 8,
"tenure_subjects": [
"architecture leadership",
"architecture leadership roles with responsibility for enterprise-wide technology direction"
]
},
{
"id": "cloud",
"status": "established",
"statement": "Cloud architecture; Azure-focused architecture and implementation.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"cloud architecture",
"cloud architectures",
"Azure",
"cloud strategy"
]
},
{
"id": "other_clouds",
"status": "established",
"statement": "Familiarity with AWS and GCP; specialist depth is not established.",
"source": "Owner-supplied W1-021 work order",
"level": "familiarity",
"aliases": [
"AWS",
"GCP"
]
},
{
"id": "infrastructure",
"status": "established",
"statement": "Infrastructure architecture.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"infrastructure architecture"
]
},
{
"id": "integration",
"status": "established",
"statement": "Enterprise integration patterns.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"enterprise integration"
]
},
{
"id": "modernization",
"status": "established",
"statement": "Technology modernization.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"modernization"
]
},
{
"id": "saas",
"status": "established",
"statement": "SaaS platform evaluation and integration.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"SaaS platform evaluation",
"SaaS integration"
]
},
{
"id": "data",
"status": "established",
"statement": "Data platforms and data architecture.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"data architecture",
"data platforms"
]
},
{
"id": "security",
"status": "established",
"statement": "Security architecture.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"security architecture"
]
},
{
"id": "governance",
"status": "established",
"statement": "Technology governance and architecture decision documentation.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"technology governance",
"architecture governance",
"architecture decision"
]
},
{
"id": "enterprise_strategy",
"status": "established",
"statement": "Enterprise technology strategy and architecture technology direction.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"enterprise technology strategy",
"enterprise architecture strategy",
"technology roadmaps"
]
},
{
"id": "communication",
"status": "established",
"statement": "Executive and stakeholder communication.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"executive communication",
"stakeholder communication",
"executive-level communication"
]
},
{
"id": "consulting",
"status": "established",
"statement": "Extensive consulting and staff augmentation for large enterprises; global-scale architecture at Deloitte; fractional CIO / technology advisory through Fractional Insight CIO LLC.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"enterprise consulting",
"staff augmentation",
"fractional CIO",
"technology advisory"
]
},
{
"id": "ai",
"status": "established",
"statement": "AI strategy and architecture, Generative AI, LLMs, RAG, local LLM architecture, AI governance concepts, Copilot-class tools, and AI-enabled knowledge and information systems.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"AI strategy",
"AI architecture",
"Generative AI",
"Large Language Models",
"LLMs",
"Retrieval-Augmented Generation",
"RAG",
"local LLM",
"AI governance",
"Copilot"
]
},
{
"id": "healthcare_ai",
"status": "established",
"statement": "AI work involving healthcare systems and HIPAA-aligned environments.",
"source": "Owner-supplied W1-021 work order",
"level": "experience",
"aliases": [
"healthcare AI",
"HIPAA-aligned"
]
},
{
"id": "past_clients",
"status": "established",
"statement": "Past enterprise clients: Kraft, BP, Allstate, Blue Cross Blue Shield, Baxter.",
"source": "Owner-supplied W1-021 work order",
"level": "context_only",
"aliases": []
},
{
"id": "ml_training",
"status": "unknown",
"statement": "Deep ML model training and data-science specialization are not established.",
"source": "Owner-supplied W1-021 work order",
"level": "unknown",
"aliases": [
"PyTorch",
"TensorFlow",
"model training"
]
},
{
"id": "mlops",
"status": "unknown",
"statement": "Production MLOps depth is not established.",
"source": "Owner-supplied W1-021 work order",
"level": "unknown",
"aliases": [
"MLOps"
]
},
{
"id": "industry_credentials",
"status": "unknown",
"statement": "Mandatory industry credentials not separately supplied remain unknown.",
"source": "Owner-supplied W1-021 work order",
"level": "unknown",
"aliases": []
}
]
}
docs/OWNER-PROFILE-CALIBRATION.md
+108
View File
@@ -0,0 +1,108 @@
# W1-021 owner profile calibration
The owner-supplied W1-021 work order is the authority for this calibration.
No career facts were inferred from browsing, clients, or job descriptions.
The profile remains data/qualification-profile.json, editable as ordinary JSON.
## Maintaining the profile
Schema version 2 adds career_evidence; version 1 profiles still load.
Each evidence item has a stable id, status, statement, source, level and aliases.
- established: explicitly supplied owner evidence.
- unknown: unverified, never a claim of absence or inability.
- excluded: an explicitly supplied owner preference against matching work.
No such career exclusions are initially set.
- aliases: phrases that identify the supported aspect of a source sentence.
They do not establish other clauses or specialist depth.
- level: experience, familiarity, context_only, or an explanatory level.
AWS and GCP are familiarity only. Past clients are context_only with no
automatic matching aliases; client history does not prove industry credentials.
- years_supported and tenure_subjects: conservative qualification thresholds
and the complete requirement subjects to which they apply. Punctuation and
standard requirement prefixes are normalized, but extra specialty clauses are
not discarded. Never add a subject merely to make a particular job pass.
More than 30 IT years is represented as a conservative 30-year threshold, and
more than 15 architecture years as 15. Eight years architecture leadership is
accepted because W1-021 explicitly directs that requirement to be supported;
it is not inferred by converting all architecture tenure into leadership tenure.
No source requisition ID is used by the matching rules.
Keep citizenship, permanent authorization, clearance, relocation, travel,
certifications and unestablished specialist depth unknown until supplied.
An empty credential list means none established in the profile, not proof of
ineligibility. In the existing personal-fact fields, yes means confirmed,
unknown means unverified, and no means an explicit owner fact or preference.
Geographic policy and role-family preferences remain separate from ability.
Record the owner source and scope when adding evidence. Broader phrases must
not be used to clear narrow tool, certification, industry or tenure requirements.
The profile is a conservative evidence ledger, not a complete CV evaluator.
## Audit semantics
qualification_evidence records source_requirement, status, scope and owner
evidence provenance. supported clears only a fully corresponding tenure gap.
supported_aspect records useful capability evidence without asserting that the
whole sentence is satisfied. unknown requires verification; mismatch is reserved
for explicit conflicting owner facts/preferences. Geographic and role-policy
mismatches remain separately explained in the existing disposition/reasons.
The specialist family policy is unchanged: specialist_mismatch is a preference
mismatch, not proof that Ken lacks skills. ML training, Salesforce credentials,
Cisco Meraki and other unestablished specialties retain verification gaps.
Known Azure experience never implies Azure certification. Desired certifications
remain desired and unknown, not required and absent.
The extractor now recognizes plain Qualifications headings and additional
explicit travel wording. Candidate credential questions exclude application
certification, system certification testing, compensation and partner-enablement
boilerplate. Original source records and all description parts remain in audit.json.
Unknown requirements outside these conservative rules still require human review.
## Offline execution and comparison
From the repository root:
python -m unittest discover -s tests -v
python w1.py practical --run runs/2026-09-17/lrs-162640 --output runs/2026-09-17/lrs-162640/qualification-W1-021 --compare-to runs/2026-09-17/lrs-162640/qualification-W1-019
Use the installed interpreter described in README.md if python is not on PATH.
Choose a fresh output directory on replay; existing evidence is never replaced.
The usual profile snapshot, summary, practical assessments, full audit and brief
are accompanied by:
- owner-verification.md: the complete deduplicated question list, including
advertised credential wording.
- owner-verification.json: every affected source ID, original URL, exact
requirement, required/desired label and disposition.
- comparison.json: before/after counts, changed dispositions and changed gaps.
Comparison requires the same saved input hashes, pipeline hash and candidate
IDs. The summary records the comparison baseline hash and all rule hashes.
Question counts include withheld and already-reviewed pipeline jobs. A question
may be relevant to the source requirements without making that opportunity
actionable. The current-reviewable count shows prioritization. Clearance and
relocation stay unknown in the profile but are not asked without applicable
source evidence.
## Verified result
All 87 saved jobs replayed offline. Disposition counts remain:
0 practical_match, 4 needs_owner_review, 36 out_of_scope, 11 location_conflict,
34 specialist_mismatch, 2 insufficient_evidence. Three of the four reviewable
jobs are existing pipeline records, leaving one new review candidate.
Requisition 46277 remains needs_owner_review. Both tenure gaps are resolved
with established architecture and leadership evidence; Azure/cloud, enterprise
strategy and modernization have aspect-level support. Permanent authorization
and four desired certifications remain unknown: Azure Solutions Architect
Expert, AWS Solutions Architect Professional/Associate, TOGAF and FinOps.
No other disposition changes. More complete requirement extraction changes
some audit gaps without overriding existing role or geographic exclusions.
Source pay, employment-type and historical metadata conflicts remain unresolved.
No live collection, applications, outreach, pipeline promotion, Git commit or
push is part of this increment.
docs/PRACTICAL-QUALIFICATION.md
+6 -3
View File
@@ -1,4 +1,4 @@
# W1-019 practical qualification
# W1-019 practical qualification
Run from the checkout root with Python 3.10+ (standard library only):
@@ -34,8 +34,8 @@ the exact interpreted profile and input/code hashes so decisions are auditable.
An in-scope Tennessee opportunity does not establish Tennessee residence.
Relocation willingness does not override the explicit approved-state policy.
Total tenure cannot establish years in a specific role: those requirements
remain verification gaps. Unknown or unlisted qualifications are not failures.
Total tenure alone cannot establish years in a specific role. W1-021 adds
explicitly scoped career evidence; see OWNER-PROFILE-CALIBRATION.md. Unknown or unlisted qualifications are not failures.
Clearance and certifications still require verification of level/current status.
## Decision rules
@@ -88,3 +88,6 @@ Review the audit when expanding owner policy or encountering new wording.
No collection, application, outreach, pipeline promotion or external commitment
is performed. Generated runs remain local under the existing Git evidence policy.
W1-021 calibration, evidence semantics and updated replay procedure:
[Owner profile calibration](OWNER-PROFILE-CALIBRATION.md).
src/opportunity_intelligence/qualification/evidence.py
+274
View File
@@ -0,0 +1,274 @@
"""Match owner evidence conservatively and render deduplicated verification needs."""
import re
def normalize(text):
"""Normalize punctuation only; preserve every substantive requirement word."""
return " ".join(re.findall(r"[a-z0-9]+", text.casefold()))
def validate(profile):
"""Reject ambiguous evidence entries rather than silently trusting them."""
entries = profile.get("career_evidence", [])
if not isinstance(entries, list):
raise ValueError("career_evidence must be a list")
seen = set()
for entry in entries:
if not isinstance(entry, dict):
raise ValueError("career evidence entries must be objects")
key = entry["id"]
if not isinstance(key, str) or not key or key in seen:
raise ValueError("career evidence IDs must be unique nonempty strings")
seen.add(key)
if entry["status"] not in {"established", "unknown", "excluded"}:
raise ValueError("invalid career evidence status")
for field in ("statement", "source", "level"):
if not isinstance(entry[field], str) or not entry[field]:
raise ValueError("evidence requires " + field)
for field in ("aliases", "tenure_subjects"):
values = entry.get(field, [])
if not isinstance(values, list) or not all(
isinstance(value, str) and value.strip() for value in values
):
raise ValueError(field + " must contain nonempty strings")
if "years_supported" in entry:
if type(entry["years_supported"]) is not int or entry["years_supported"] < 0:
raise ValueError("years_supported must be a nonnegative integer")
if not entry.get("tenure_subjects"):
raise ValueError("tenure evidence requires explicit subjects")
if profile["schema_version"] == 2 and not entries:
raise ValueError("version 2 requires career_evidence")
def is_personal_credential(text):
"""Separate candidate credentials from product testing and employer prose."""
if not re.search(r"certification|certified|credential|licen[sc]e", text, re.I):
return False
if re.fullmatch(r"(?:certifications?|the following certifications are highly valued):?", text.strip(), re.I):
return False
return not re.search(
r"application certification|certification (?:testing|processes|programs for)|"
r"partner success|compensation decisions|salary.{0,50}certifications",
text, re.I,
)
def tenure_support(requirement, profile):
"""Match an entire tenure subject; broad IT years cannot prove tool depth."""
match = re.fullmatch(
r"(?:must have\s+|minimum\s+(?:of\s+)?|at least\s+|required:\s*)?"
r"(\d+)\+?\s+years?\s*"
r"(?:(?:of\s+)?experience\s*)?"
r"(?:(?:with the following|in|with|of)\s*:?\s*)?"
r"(.+?)[.]?",
requirement.strip(), re.I,
)
if not match:
return []
years, subject = int(match[1]), normalize(match[2])
return [
entry for entry in profile.get("career_evidence", [])
if entry["status"] == "established"
and entry.get("years_supported", -1) >= years
and subject in {normalize(value) for value in entry.get("tenure_subjects", [])}
]
def owner_reference(entry):
"""Copy provenance into the audit so it remains interpretable independently."""
return {key: entry[key] for key in ("id", "status", "statement", "source", "level")}
def calibrate(parts, gaps, profile):
"""Resolve supported tenure gaps and record aspect-level career evidence.
An aspect match never claims that a compound sentence is fully satisfied.
Only exact normalized tenure subjects remove an existing experience gap.
Certifications, specialist skills and personal facts are never cleared here.
"""
remaining = []
decisions = []
for gap in gaps:
requirement = gap["evidence"][0]
support = (
tenure_support(requirement, profile)
if gap["fact"] == "role_specific_experience" else []
)
if support:
decisions.append({
"source_requirement": requirement,
"status": "supported",
"scope": "complete tenure requirement",
"owner_evidence": [owner_reference(entry) for entry in support],
})
else:
remaining.append(gap)
for requirement in gap["evidence"]:
decisions.append({
"source_requirement": requirement,
"status": "mismatch" if gap["owner_value"] == "no" else "unknown",
"scope": gap.get("skill", gap["fact"]),
"requirement_level": gap.get("requirement_level", "required"),
"owner_evidence": [],
"owner_value": gap["owner_value"],
})
for part in parts:
normalized = " " + normalize(part) + " "
for entry in profile.get("career_evidence", []):
if entry["status"] != "established":
continue
matched = [
alias for alias in entry["aliases"]
if " " + normalize(alias) + " " in normalized
]
if matched:
decisions.append({
"source_requirement": part,
"status": "supported_aspect",
"scope": "matched aspect only; other clauses remain unverified",
"matched_aspects": matched,
"owner_evidence": [owner_reference(entry)],
})
return remaining, decisions
def excluded_requirements(parts, profile):
"""Apply only explicitly excluded owner preferences, never unknown evidence."""
return [
{"source_requirement": part, "status": "mismatch",
"scope": "explicit owner exclusion",
"owner_evidence": [owner_reference(entry)]}
for part in parts
for entry in profile.get("career_evidence", [])
if entry["status"] == "excluded"
and any(
" " + normalize(alias) + " " in " " + normalize(part) + " "
for alias in entry["aliases"]
)
]
QUESTIONS = {
"us_work_authorization_without_sponsorship":
"Do you have permanent US work authorization for any employer without sponsorship?",
"us_citizenship": "Do you meet the advertised US citizenship requirements?",
"security_clearance": "What active clearance, level and eligibility can you verify?",
"willing_to_travel": "What travel frequency and destinations would you accept?",
"certifications": "Which advertised certifications or credentials do you currently hold?",
"role_specific_experience":
"Which remaining role-specific tenure requirements can you substantiate?",
"residence_state": "What is your residence state, and which advertised residency restrictions can you meet?",
}
def verification_list(records):
"""Group repeated unknown facts while retaining all affected job evidence."""
groups = {}
for record in records:
gaps = list(record["verification_gaps"])
for restriction in record["geography"]["residency_evidence"]:
if record["geography"]["gaps"]:
gaps.append({
"fact": "residence_state", "owner_value": "unknown",
"evidence": [restriction],
})
for gap in gaps:
if gap["owner_value"] == "no":
continue
key = gap.get("skill", gap["fact"])
question = QUESTIONS.get(
key, "What hands-on depth can you establish in " + key + "?"
)
group = groups.setdefault(key, {
"fact": key, "question": question, "opportunities": {},
})
opportunity = group["opportunities"].setdefault(record["source_id"], {
"source_id": record["source_id"], "title": record["title"],
"url": record["url"], "disposition": record["disposition"],
"existing_pipeline_record": bool(record["existing_pipeline_records"]),
"requirements": [],
})
for text in gap["evidence"]:
item = {
"text": text,
"level": gap.get("requirement_level", "required"),
}
if item not in opportunity["requirements"]:
opportunity["requirements"].append(item)
result = []
for key in sorted(groups):
group = groups[key]
group["opportunities"] = sorted(
group["opportunities"].values(), key=lambda item: int(item["source_id"])
)
group["affected_count"] = len(group["opportunities"])
group["reviewable_count"] = sum(
item["disposition"] in {"practical_match", "needs_owner_review"}
for item in group["opportunities"]
)
result.append(group)
return result
def render_verification(groups):
"""Render the complete question list; JSON retains every source requirement."""
lines = [
"# W1-021 owner verification", "",
"Questions below arise from saved jobs, not assumptions about the owner.",
"Counts include withheld and existing pipeline jobs; answering a question",
"does not remove independent location or role-policy exclusions.",
"See owner-verification.json for each original link, exact requirement,",
"required/desired label and current disposition.", "",
]
for group in groups:
lines.append(
f"- {group['question']} "
f"({group['affected_count']} jobs; "
f"{group['reviewable_count']} currently reviewable.)"
)
if group["fact"] == "certifications":
requirements = sorted({
requirement["text"]
for opportunity in group["opportunities"]
for requirement in opportunity["requirements"]
})
lines += ["", " Advertised credential wording:", ""]
lines.extend(" - " + text for text in requirements)
return "\n".join(lines) + "\n"
def compare_results(before, after):
"""Compare complete candidate sets without hiding unchanged dispositions."""
old = {item["source_id"]: item for item in before}
new = {item["source_id"]: item for item in after}
if len(old) != len(before) or len(new) != len(after) or old.keys() != new.keys():
raise ValueError("Comparison requires identical unique candidate IDs")
changes = []
for key in sorted(old, key=int):
previous, current = old[key], new[key]
if (previous["disposition"] != current["disposition"]
or previous["verification_gaps"] != current["verification_gaps"]):
changes.append({
"source_id": key, "url": current["url"], "title": current["title"],
"before_disposition": previous["disposition"],
"after_disposition": current["disposition"],
"before_gaps": previous["verification_gaps"],
"after_gaps": current["verification_gaps"],
"after_reasons": current["reasons"],
"supported_tenure": [
item for item in current["qualification_evidence"]
if item["status"] == "supported"
],
})
return {
"candidate_count": len(after),
"disposition_changed_ids": [
item["source_id"] for item in changes
if item["before_disposition"] != item["after_disposition"]
],
"gap_or_disposition_changes": changes,
}
src/opportunity_intelligence/qualification/practical.py
+64 -14
View File
@@ -1,4 +1,4 @@
"""Practical eligibility over saved LRS evidence and owner-controlled policy."""
"""Practical eligibility over saved LRS evidence and owner-controlled policy."""
import argparse
from collections import Counter
@@ -12,9 +12,10 @@ from opportunity_intelligence.collectors.lrs import (
)
from opportunity_intelligence.paths import PIPELINE, ROOT
from opportunity_intelligence.qualification import lrs as technical
from opportunity_intelligence.qualification import evidence
VERSION = "W1-019-v1"
VERSION = "W1-021-v1"
PROFILE = ROOT / "data/qualification-profile.json"
REVIEWABLE = {"practical_match", "needs_owner_review"}
DISPOSITIONS = (
@@ -60,6 +61,7 @@ STATE_NAMES = (
STATES = dict(item.split(":") for item in STATE_NAMES.split("|"))
SPECIALIST_SKILLS = {
"Python ML frameworks": r"Python|PyTorch|TensorFlow|scikit-learn",
"ML model building": r"model.training|train.{0,25}models|built or trained|MLOps",
"Salesforce": r"Salesforce|Apex|SOQL",
"ServiceNow": r"ServiceNow|HRSD",
".NET implementation": r"C#|\.NET|Entity Framework",
@@ -88,8 +90,9 @@ def load_profile(path):
"""Validate owner policy; malformed configuration must fail visibly."""
profile = json.loads(path.read_text(encoding="utf-8-sig"))
try:
if profile["schema_version"] != 1:
raise ValueError("schema_version must be 1")
if profile["schema_version"] not in (1, 2):
raise ValueError("schema_version must be 1 or 2")
evidence.validate(profile)
categories = profile["opportunity_categories"]
allowed_categories = {
"full_time_employment", "part_time_employment", "contract", "consulting"
@@ -243,13 +246,15 @@ def owner_gaps(parts, profile):
("security_clearance",
r"(?:active|required|must).{0,40}security clearance"),
("willing_to_travel",
r"travel.{0,40}(?:required|must)|must.{0,30}travel"),
r"travel.{0,40}(?:required|requirement|must)|"
r"(?:must|requires?|ability to|will|willingness).{0,50}travel|"
r"^Travel:\s*\d"),
)
for key, pattern in checks:
evidence = [part for part in parts if matches(pattern, part)]
if evidence and facts[key] != "yes":
fact_evidence = [part for part in parts if matches(pattern, part)]
if fact_evidence and facts[key] != "yes":
gaps.append({
"fact": key, "owner_value": facts[key], "evidence": evidence,
"fact": key, "owner_value": facts[key], "evidence": fact_evidence,
})
if facts[key] == "no":
known_conflicts.append(key)
@@ -259,13 +264,13 @@ def owner_gaps(parts, profile):
if matches(r"^(Preferred Qualifications|Strong Candidates Will Have|Key Responsibilities)", part):
required_section = False
elif matches(
r"^(Required Qualifications|Requirements|Skills.*Qualifications)", part
r"^(Required Qualifications|Candidate Requirements|Qualifications|Requirements|Skills.*Qualifications)", part
):
required_section = True
mandatory = required_section or matches(
r"\bmust\b|\brequired\b|expert.level", part
)
if matches(r"certification|certified", part):
if evidence.is_personal_credential(part):
confirmed = facts["certifications"]
if confirmed == "unknown" or not any(
item.casefold() in part.casefold() for item in confirmed
@@ -302,6 +307,11 @@ def assess(record, text, parts, profile, as_of):
family_policy = profile["role_family_policy"][family]
geography = geography_assessment(record, parts, profile)
gaps, known_conflicts = owner_gaps(parts, profile)
gaps, qualification_evidence = evidence.calibrate(parts, gaps, profile)
explicit_exclusions = evidence.excluded_requirements(parts, profile)
qualification_evidence.extend(explicit_exclusions)
if explicit_exclusions:
known_conflicts.append("explicit career preference exclusion")
domain_hits = [
domain for domain in profile["strong_domains"]
if domain.casefold() in text.casefold()
@@ -357,6 +367,7 @@ def assess(record, text, parts, profile, as_of):
"role_policy": family_policy, "category": category,
"disposition": disposition, "reasons": [reason], "geography": geography,
"verification_gaps": gaps, "known_owner_conflicts": known_conflicts,
"qualification_evidence": qualification_evidence,
"source_conflicts": stage_one["conflicts"],
"source_dates": stage_one["source_dates"],
"advertised_pay": stage_one["advertised_pay"],
@@ -389,6 +400,14 @@ def render_brief(records, summary):
+ ". Advertised, not verified.",
"- Requirements: " + " | ".join(record["qualifications"]),
]
supported = sorted({
owner["statement"]
for decision in record["qualification_evidence"]
if decision["status"] in {"supported", "supported_aspect"}
for owner in decision["owner_evidence"]
})
lines.append("- Established matching evidence (aspect-level): "
+ " | ".join(supported))
# Combine repeated unknowns without losing their full audit evidence.
groups = {}
for gap in record["verification_gaps"]:
@@ -423,7 +442,8 @@ def render_brief(records, summary):
return "\n".join(lines)
def execute(run, pipeline_path, profile_path, output, limit=15):
def execute(run, pipeline_path, profile_path, output, limit=15,
compare_to=None):
"""Verify saved collection bytes and create a separate reproducible audit."""
if not 1 <= limit <= 15:
raise SourceError("Brief limit must be between 1 and 15")
@@ -498,17 +518,44 @@ def execute(run, pipeline_path, profile_path, output, limit=15):
},
"rules_sha256": {
path.name: hashlib.sha256(path.read_bytes()).hexdigest()
for path in (Path(__file__), Path(technical.__file__))
for path in (Path(__file__), Path(technical.__file__),
Path(evidence.__file__))
},
"pipeline_unchanged": pipeline_path.read_bytes() == pipeline_bytes,
}
if not summary["pipeline_unchanged"]:
raise SourceError("Pipeline changed during qualification")
comparison = None
if compare_to is not None:
baseline_summary = json.loads(
(compare_to / "summary.json").read_text(encoding="utf-8")
)
for field in ("input_sha256", "pipeline_sha256"):
if baseline_summary[field] != summary[field]:
raise SourceError("Comparison input mismatch: " + field)
baseline_bytes = (compare_to / "practical.json").read_bytes()
comparison = evidence.compare_results(
json.loads(baseline_bytes), practical
)
comparison["before_counts"] = baseline_summary[
"practical_dispositions_all_jobs"
]
comparison["after_counts"] = summary["practical_dispositions_all_jobs"]
summary["comparison_baseline_sha256"] = hashlib.sha256(
baseline_bytes
).hexdigest()
output.mkdir(parents=True, exist_ok=False)
write_json(output / "profile.json", profile)
write_json(output / "audit.json", audit)
write_json(output / "practical.json", practical)
write_json(output / "summary.json", summary)
if comparison is not None:
write_json(output / "comparison.json", comparison)
questions = evidence.verification_list(practical)
write_json(output / "owner-verification.json", questions)
(output / "owner-verification.md").write_text(
evidence.render_verification(questions), encoding="utf-8"
)
(output / "brief.md").write_text(
render_brief(reviewable[:limit], summary), encoding="utf-8"
)
@@ -523,11 +570,14 @@ def main():
parser.add_argument("--profile", type=Path, default=PROFILE)
parser.add_argument("--output", type=Path)
parser.add_argument("--limit", type=int, default=15)
parser.add_argument("--compare-to", type=Path,
help="Saved practical output directory to compare")
args = parser.parse_args()
try:
run = args.run or technical.latest_run()
output = args.output or run / "qualification-W1-019"
summary = execute(run, args.pipeline, args.profile, output, args.limit)
output = args.output or run / "qualification-W1-021"
summary = execute(run, args.pipeline, args.profile, output,
args.limit, args.compare_to)
except (OSError, ValueError, KeyError) as error:
raise SystemExit(f"ERROR: {error}") from error
print(json.dumps(summary, indent=2))
tests/test_calibration.py
+199
View File
@@ -0,0 +1,199 @@
"""Evidence calibration, unknown personal facts and conservative matching."""
import copy
import json
from pathlib import Path
import tempfile
import unittest
from context import FIXTURES
from test_qualification import record
from opportunity_intelligence.qualification import evidence, practical
class CalibrationTests(unittest.TestCase):
def setUp(self):
self.profile = practical.load_profile(practical.PROFILE)
def assess(self, parts, title="Cloud Architect", location="Remote"):
candidate = record(title=title)
candidate["location"] = [location]
return practical.assess(
candidate, " ".join(parts), parts, self.profile, "2026-09-17"
)
def test_established_architecture_years(self):
result = self.assess(["Must have 15+ years architecture experience."])
self.assertFalse(result["verification_gaps"])
self.assertEqual(result["qualification_evidence"][0]["status"], "supported")
self.assertEqual(result["qualification_evidence"][0]["owner_evidence"][0]["id"],
"architecture")
def test_established_leadership_years(self):
result = self.assess(["Must have 8+ years architecture leadership."])
self.assertFalse(result["verification_gaps"])
self.assertTrue(any(
item["status"] == "supported" for item in result["qualification_evidence"]
))
def test_azure_cloud_and_strategy_have_provenance(self):
result = self.assess([
"Required Qualifications", "Azure cloud architecture.",
"Enterprise architecture strategy and roadmaps; technology modernization."
])
ids = {
owner["id"] for decision in result["qualification_evidence"]
for owner in decision["owner_evidence"]
}
self.assertTrue({"cloud", "enterprise_strategy", "modernization"} <= ids)
self.assertFalse(result["verification_gaps"])
def test_work_authorization_stays_unknown(self):
result = self.assess([
"Azure architecture.", "Must have permanent authorization to work in the USA for any employer."
])
self.assertEqual(result["disposition"], "needs_owner_review")
self.assertEqual(result["verification_gaps"][0]["owner_value"], "unknown")
def test_certification_stays_unknown_even_with_cloud_experience(self):
result = self.assess([
"Azure architecture.", "Preferred Qualifications",
"Microsoft Certified: Azure Solutions Architect Expert"
])
self.assertEqual(result["verification_gaps"][0]["fact"], "certifications")
self.assertEqual(result["verification_gaps"][0]["requirement_level"],
"desired_or_unspecified")
self.assertEqual(result["known_owner_conflicts"], [])
def test_ml_model_training_is_not_inferred_from_ai(self):
result = self.assess([
"AI architecture. Must have actually built or trained ML models.",
"Required Qualifications", "PyTorch and TensorFlow required."
], title="AI/ML Platform Architect")
self.assertEqual(result["disposition"], "specialist_mismatch")
skills = {gap.get("skill") for gap in result["verification_gaps"]}
self.assertTrue({"ML model building", "Python ML frameworks"} <= skills)
self.assertFalse(result["known_owner_conflicts"])
def test_compound_tenure_does_not_clear_specialist_requirement(self):
result = self.assess([
"Must have 15+ years architecture experience and PyTorch model training."
])
self.assertIn("role_specific_experience",
{gap["fact"] for gap in result["verification_gaps"]})
def test_insufficient_tenure_evidence_is_unknown_not_mismatch(self):
result = self.assess(["Must have 25+ years architecture experience."])
self.assertTrue(result["verification_gaps"])
self.assertFalse(result["known_owner_conflicts"])
def test_cisco_specialization_is_unknown(self):
result = self.assess([
"Qualifications", "5+ years of experience supporting Cisco Meraki networking solutions.",
"Azure architecture."
], title="Senior Network Engineer")
self.assertEqual(result["disposition"], "specialist_mismatch")
self.assertIn("Cisco Meraki",
{gap.get("skill") for gap in result["verification_gaps"]})
def test_relocation_not_inferred(self):
result = self.assess(
["Azure architecture.", "This role is onsite in Chicago, IL."],
location="Chicago, IL",
)
self.assertEqual(result["disposition"], "location_conflict")
self.assertEqual(self.profile["owner_facts"]["willing_to_relocate"], "unknown")
def test_explicit_exclusion_is_distinct_from_unknown(self):
self.profile["career_evidence"].append({
"id": "future_owner_preference", "status": "excluded",
"statement": "Owner excludes this work.", "source": "test-only owner instruction",
"level": "preference", "aliases": ["cloud architecture"],
})
result = self.assess(["Cloud architecture required."])
self.assertEqual(result["disposition"], "out_of_scope")
self.assertTrue(result["known_owner_conflicts"])
def test_credential_boilerplate_does_not_ask_personal_question(self):
result = self.assess([
"Azure architecture.", "Assist with application certification and deployment.",
"Compensation decisions are based on certifications and market considerations."
])
self.assertFalse(result["verification_gaps"])
def test_aws_familiarity_never_satisfies_specialist_tenure(self):
result = self.assess(["Must have 10+ years AWS CDK experience."])
self.assertTrue(result["verification_gaps"])
self.assertFalse(any(
item["status"] == "supported" for item in result["qualification_evidence"]
))
def test_invalid_evidence_and_legacy_profile(self):
legacy = copy.deepcopy(self.profile)
legacy["schema_version"] = 1
del legacy["career_evidence"]
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "profile.json"
path.write_text(json.dumps(legacy))
self.assertEqual(practical.load_profile(path)["schema_version"], 1)
broken = copy.deepcopy(self.profile)
broken["career_evidence"][0]["status"] = "assumed"
path.write_text(json.dumps(broken))
with self.assertRaises(ValueError):
practical.load_profile(path)
def test_comparison_requires_same_candidates(self):
with self.assertRaises(ValueError):
evidence.compare_results([{"source_id": "1"}], [])
def test_salesforce_credentials_and_depth_remain_unknown(self):
result = self.assess([
"Azure architecture.", "Required Qualifications",
"Required - Minimum of 2 Salesforce Architect Certifications",
], title="Salesforce Architect")
self.assertEqual(result["disposition"], "specialist_mismatch")
self.assertEqual({gap["fact"] for gap in result["verification_gaps"]},
{"certifications", "specialist_skills"})
def test_saved_replay_supports_tenure_and_preserves_pipeline(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
pipeline = root / "pipeline.json"
original = (FIXTURES / "opportunities.json").read_bytes()
pipeline.write_bytes(original)
first, second = root / "first", root / "second"
for output in (first, second):
practical.execute(FIXTURES / "lrs-run", pipeline,
practical.PROFILE, output)
self.assertEqual(original, pipeline.read_bytes())
for file in first.iterdir():
self.assertEqual(file.read_bytes(), (second / file.name).read_bytes())
compared = root / "compared"
practical.execute(FIXTURES / "lrs-run", pipeline, practical.PROFILE,
compared, compare_to=first)
comparison = json.loads(
(compared / "comparison.json").read_text(encoding="utf-8")
)
self.assertEqual(comparison["disposition_changed_ids"], [])
self.assertEqual(comparison["gap_or_disposition_changes"], [])
rows = json.loads((first / "practical.json").read_text(encoding="utf-8"))
target = next(row for row in rows if row["source_id"] == "46277")
self.assertEqual(target["disposition"], "needs_owner_review")
self.assertEqual(sum(
item["status"] == "supported" for item in target["qualification_evidence"]
), 2)
self.assertNotIn("role_specific_experience",
{gap["fact"] for gap in target["verification_gaps"]})
questions = json.loads(
(first / "owner-verification.json").read_text(encoding="utf-8")
)
keys = [question["fact"] for question in questions]
self.assertEqual(len(keys), len(set(keys)))
self.assertNotIn("security_clearance", keys)
self.assertTrue(all(question["affected_count"] > 0 for question in questions))
if __name__ == "__main__":
unittest.main()